2023-09-29 06:12:51,063 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@10aa41f2 2023-09-29 06:12:51,084 main DEBUG Took 0.018282 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2023-09-29 06:12:51,084 main DEBUG PluginManager 'Core' found 129 plugins 2023-09-29 06:12:51,085 main DEBUG PluginManager 'Level' found 0 plugins 2023-09-29 06:12:51,086 main DEBUG PluginManager 'Lookup' found 16 plugins 2023-09-29 06:12:51,088 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-29 06:12:51,098 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2023-09-29 06:12:51,115 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-29 06:12:51,117 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-29 06:12:51,119 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-29 06:12:51,119 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-29 06:12:51,120 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-29 06:12:51,120 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-29 06:12:51,121 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-29 06:12:51,121 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-29 06:12:51,122 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-29 06:12:51,122 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-29 06:12:51,124 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-29 06:12:51,124 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-29 06:12:51,125 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-29 06:12:51,126 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-29 06:12:51,126 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-29 06:12:51,126 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-29 06:12:51,127 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-29 06:12:51,127 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-29 06:12:51,127 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-29 06:12:51,128 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-29 06:12:51,128 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-29 06:12:51,128 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-29 06:12:51,129 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-29 06:12:51,129 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-29 06:12:51,130 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-29 06:12:51,130 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2023-09-29 06:12:51,132 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-29 06:12:51,135 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2023-09-29 06:12:51,137 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2023-09-29 06:12:51,137 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2023-09-29 06:12:51,138 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2023-09-29 06:12:51,138 main DEBUG PluginManager 'Converter' found 47 plugins 2023-09-29 06:12:51,151 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2023-09-29 06:12:51,154 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2023-09-29 06:12:51,155 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2023-09-29 06:12:51,156 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2023-09-29 06:12:51,156 main DEBUG createAppenders(={Console}) 2023-09-29 06:12:51,157 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@10aa41f2 initialized 2023-09-29 06:12:51,158 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@10aa41f2 2023-09-29 06:12:51,158 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@10aa41f2 OK. 2023-09-29 06:12:51,159 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2023-09-29 06:12:51,159 main DEBUG OutputStream closed 2023-09-29 06:12:51,159 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2023-09-29 06:12:51,160 main DEBUG Appender DefaultConsole-1 stopped with status true 2023-09-29 06:12:51,160 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@3023df74 OK 2023-09-29 06:12:51,220 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586 2023-09-29 06:12:51,223 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=StatusLogger 2023-09-29 06:12:51,224 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=ContextSelector 2023-09-29 06:12:51,226 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name= 2023-09-29 06:12:51,226 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.directory 2023-09-29 06:12:51,227 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2023-09-29 06:12:51,227 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.zookeeper 2023-09-29 06:12:51,227 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2023-09-29 06:12:51,227 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2023-09-29 06:12:51,228 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2023-09-29 06:12:51,228 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase 2023-09-29 06:12:51,228 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop 2023-09-29 06:12:51,229 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2023-09-29 06:12:51,229 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2023-09-29 06:12:51,229 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2023-09-29 06:12:51,229 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2023-09-29 06:12:51,230 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2023-09-29 06:12:51,231 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Appenders,name=Console 2023-09-29 06:12:51,237 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-09-29 06:12:51,238 main DEBUG Reconfiguration complete for context[name=1b6d3586] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.6.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@54a7079e) with optional ClassLoader: null 2023-09-29 06:12:51,238 main DEBUG Shutdown hook enabled. Registering a new one. 2023-09-29 06:12:51,239 main DEBUG LoggerContext[name=1b6d3586, org.apache.logging.log4j.core.LoggerContext@54a7079e] started OK. 2023-09-29T06:12:51,545 DEBUG [main {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a22ba870-907c-9d99-3f64-a164cf019894 2023-09-29 06:12:51,549 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2023-09-29 06:12:51,549 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-09-29T06:12:51,560 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithAdaptivePolicy timeout: 13 mins 2023-09-29T06:12:51,572 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2023-09-29T06:12:51,577 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a22ba870-907c-9d99-3f64-a164cf019894/cluster_925cee83-985f-a2ad-bbe0-74639e9c7977, deleteOnExit=true 2023-09-29T06:12:51,577 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2023-09-29T06:12:51,578 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a22ba870-907c-9d99-3f64-a164cf019894/test.cache.data in system properties and HBase conf 2023-09-29T06:12:51,579 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a22ba870-907c-9d99-3f64-a164cf019894/hadoop.tmp.dir in system properties and HBase conf 2023-09-29T06:12:51,579 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a22ba870-907c-9d99-3f64-a164cf019894/hadoop.log.dir in system properties and HBase conf 2023-09-29T06:12:51,580 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a22ba870-907c-9d99-3f64-a164cf019894/mapreduce.cluster.local.dir in system properties and HBase conf 2023-09-29T06:12:51,581 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a22ba870-907c-9d99-3f64-a164cf019894/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-09-29T06:12:51,581 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-09-29T06:12:51,712 WARN [Time-limited test {}] util.NativeCodeLoader(62): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2023-09-29T06:12:52,177 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2023-09-29T06:12:52,181 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a22ba870-907c-9d99-3f64-a164cf019894/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-09-29T06:12:52,182 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a22ba870-907c-9d99-3f64-a164cf019894/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-09-29T06:12:52,182 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a22ba870-907c-9d99-3f64-a164cf019894/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-09-29T06:12:52,183 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a22ba870-907c-9d99-3f64-a164cf019894/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-09-29T06:12:52,183 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a22ba870-907c-9d99-3f64-a164cf019894/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-09-29T06:12:52,183 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a22ba870-907c-9d99-3f64-a164cf019894/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-09-29T06:12:52,184 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a22ba870-907c-9d99-3f64-a164cf019894/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-09-29T06:12:52,184 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a22ba870-907c-9d99-3f64-a164cf019894/dfs.journalnode.edits.dir in system properties and HBase conf 2023-09-29T06:12:52,184 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a22ba870-907c-9d99-3f64-a164cf019894/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-09-29T06:12:52,185 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a22ba870-907c-9d99-3f64-a164cf019894/nfs.dump.dir in system properties and HBase conf 2023-09-29T06:12:52,185 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a22ba870-907c-9d99-3f64-a164cf019894/java.io.tmpdir in system properties and HBase conf 2023-09-29T06:12:52,185 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a22ba870-907c-9d99-3f64-a164cf019894/dfs.journalnode.edits.dir in system properties and HBase conf 2023-09-29T06:12:52,185 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a22ba870-907c-9d99-3f64-a164cf019894/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-09-29T06:12:52,186 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a22ba870-907c-9d99-3f64-a164cf019894/fs.s3a.committer.staging.tmp.path in system properties and HBase conf Formatting using clusterid: testClusterID 2023-09-29T06:12:52,723 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-09-29T06:12:52,728 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-09-29T06:12:53,058 WARN [Time-limited test {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2023-09-29T06:12:53,201 INFO [Time-limited test {}] log.Slf4jLog(67): Logging to org.apache.logging.slf4j.Log4jLogger@23b354c9 via org.mortbay.log.Slf4jLog 2023-09-29T06:12:53,221 WARN [Time-limited test {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-09-29T06:12:53,268 INFO [Time-limited test {}] log.Slf4jLog(67): jetty-6.1.26 2023-09-29T06:12:53,308 INFO [Time-limited test {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/hdfs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a22ba870-907c-9d99-3f64-a164cf019894/java.io.tmpdir/Jetty_localhost_44167_hdfs____7q2ifw/webapp 2023-09-29T06:12:53,480 INFO [Time-limited test {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:44167 2023-09-29T06:12:53,515 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-09-29T06:12:53,515 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-09-29T06:12:54,186 WARN [Listener at localhost/34233 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-09-29T06:12:54,262 WARN [Listener at localhost/34233 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-09-29T06:12:54,283 WARN [Listener at localhost/34233 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-09-29T06:12:54,292 INFO [Listener at localhost/34233 {}] log.Slf4jLog(67): jetty-6.1.26 2023-09-29T06:12:54,299 INFO [Listener at localhost/34233 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a22ba870-907c-9d99-3f64-a164cf019894/java.io.tmpdir/Jetty_localhost_39843_datanode____.4t0snc/webapp 2023-09-29T06:12:54,410 INFO [Listener at localhost/34233 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:39843 2023-09-29T06:12:54,660 WARN [Listener at localhost/42843 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-09-29T06:12:55,008 WARN [Thread-52 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-09-29T06:12:55,254 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xf8fdf8950a6e27e1: Processing first storage report for DS-2e154dbb-27b5-42a6-ae9b-87cdc4287cdf from datanode 3d6298e7-fdaf-4ed5-963c-adc3ebb4292e 2023-09-29T06:12:55,257 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xf8fdf8950a6e27e1: from storage DS-2e154dbb-27b5-42a6-ae9b-87cdc4287cdf node DatanodeRegistration(127.0.0.1:42427, datanodeUuid=3d6298e7-fdaf-4ed5-963c-adc3ebb4292e, infoPort=42279, infoSecurePort=0, ipcPort=42843, storageInfo=lv=-57;cid=testClusterID;nsid=45335293;c=1695967972816), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2023-09-29T06:12:55,257 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xf8fdf8950a6e27e1: Processing first storage report for DS-29ea5562-60e4-4bbb-b8e7-eac08b9465a9 from datanode 3d6298e7-fdaf-4ed5-963c-adc3ebb4292e 2023-09-29T06:12:55,257 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xf8fdf8950a6e27e1: from storage DS-29ea5562-60e4-4bbb-b8e7-eac08b9465a9 node DatanodeRegistration(127.0.0.1:42427, datanodeUuid=3d6298e7-fdaf-4ed5-963c-adc3ebb4292e, infoPort=42279, infoSecurePort=0, ipcPort=42843, storageInfo=lv=-57;cid=testClusterID;nsid=45335293;c=1695967972816), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-09-29T06:12:55,295 DEBUG [Listener at localhost/42843 {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a22ba870-907c-9d99-3f64-a164cf019894 2023-09-29T06:12:55,374 INFO [Listener at localhost/42843 {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a22ba870-907c-9d99-3f64-a164cf019894/cluster_925cee83-985f-a2ad-bbe0-74639e9c7977/zookeeper_0, clientPort=49344, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a22ba870-907c-9d99-3f64-a164cf019894/cluster_925cee83-985f-a2ad-bbe0-74639e9c7977/zookeeper_0/version-2, dataDirSize=424 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a22ba870-907c-9d99-3f64-a164cf019894/cluster_925cee83-985f-a2ad-bbe0-74639e9c7977/zookeeper_0/version-2, dataLogSize=424 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, serverId=0 2023-09-29T06:12:55,393 INFO [Listener at localhost/42843 {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=49344 2023-09-29T06:12:55,401 INFO [Listener at localhost/42843 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-29T06:12:55,403 INFO [Listener at localhost/42843 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-29T06:12:56,140 INFO [Listener at localhost/42843 {}] util.FSUtils(462): Created version file at hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a with version=8 2023-09-29T06:12:56,141 INFO [Listener at localhost/42843 {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/hbase-staging 2023-09-29T06:12:56,369 DEBUG [Listener at localhost/42843 {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2023-09-29T06:12:57,137 INFO [Listener at localhost/42843 {}] client.ConnectionUtils(128): master/jenkins-hbase3:0 server-side Connection retries=45 2023-09-29T06:12:57,175 INFO [Listener at localhost/42843 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-09-29T06:12:57,176 INFO [Listener at localhost/42843 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-09-29T06:12:57,176 INFO [Listener at localhost/42843 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-09-29T06:12:57,176 INFO [Listener at localhost/42843 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-09-29T06:12:57,177 INFO [Listener at localhost/42843 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-09-29T06:12:57,357 INFO [Listener at localhost/42843 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-09-29T06:12:57,472 INFO [Listener at localhost/42843 {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2023-09-29T06:12:57,494 DEBUG [Listener at localhost/42843 {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2023-09-29T06:12:57,502 INFO [Listener at localhost/42843 {}] ipc.NettyRpcServer(310): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-09-29T06:12:57,561 DEBUG [Listener at localhost/42843 {}] channel.DefaultChannelId(79): -Dio.netty.processId: 23284 (auto-detected) 2023-09-29T06:12:57,563 DEBUG [Listener at localhost/42843 {}] channel.DefaultChannelId(101): -Dio.netty.machineId: 02:42:1c:ff:fe:e2:13:ea (auto-detected) 2023-09-29T06:12:57,611 INFO [Listener at localhost/42843 {}] ipc.NettyRpcServer(192): Bind to /172.31.12.81:42173 2023-09-29T06:12:57,625 INFO [Listener at localhost/42843 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-29T06:12:57,628 INFO [Listener at localhost/42843 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-29T06:12:57,651 INFO [Listener at localhost/42843 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=master:42173 connecting to ZooKeeper ensemble=127.0.0.1:49344 2023-09-29T06:12:57,972 DEBUG [Listener at localhost/42843-EventThread {}] zookeeper.ZKWatcher(604): master:421730x0, quorum=127.0.0.1:49344, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-09-29T06:12:57,978 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): master:42173-0x101625af8180000 connected 2023-09-29T06:12:58,034 DEBUG [Listener at localhost/42843 {}] zookeeper.ZKUtil(113): master:42173-0x101625af8180000, quorum=127.0.0.1:49344, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2023-09-29T06:12:58,038 DEBUG [Listener at localhost/42843 {}] zookeeper.ZKUtil(113): master:42173-0x101625af8180000, quorum=127.0.0.1:49344, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2023-09-29T06:12:58,042 DEBUG [Listener at localhost/42843 {}] zookeeper.ZKUtil(113): master:42173-0x101625af8180000, quorum=127.0.0.1:49344, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2023-09-29T06:12:58,053 DEBUG [Listener at localhost/42843 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42173 2023-09-29T06:12:58,053 DEBUG [Listener at localhost/42843 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42173 2023-09-29T06:12:58,053 DEBUG [Listener at localhost/42843 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42173 2023-09-29T06:12:58,054 DEBUG [Listener at localhost/42843 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42173 2023-09-29T06:12:58,054 DEBUG [Listener at localhost/42843 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42173 2023-09-29T06:12:58,064 INFO [Listener at localhost/42843 {}] master.HMaster(485): hbase.rootdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a, hbase.cluster.distributed=false 2023-09-29T06:12:58,139 INFO [Listener at localhost/42843 {}] client.ConnectionUtils(128): regionserver/jenkins-hbase3:0 server-side Connection retries=45 2023-09-29T06:12:58,139 INFO [Listener at localhost/42843 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-09-29T06:12:58,140 INFO [Listener at localhost/42843 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-09-29T06:12:58,140 INFO [Listener at localhost/42843 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-09-29T06:12:58,140 INFO [Listener at localhost/42843 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-09-29T06:12:58,140 INFO [Listener at localhost/42843 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-09-29T06:12:58,145 INFO [Listener at localhost/42843 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-09-29T06:12:58,148 INFO [Listener at localhost/42843 {}] ipc.NettyRpcServer(310): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-09-29T06:12:58,149 INFO [Listener at localhost/42843 {}] ipc.NettyRpcServer(192): Bind to /172.31.12.81:39783 2023-09-29T06:12:58,152 INFO [Listener at localhost/42843 {}] hfile.BlockCacheFactory(127): Allocating BlockCache size=782.40 MB, blockSize=64 KB 2023-09-29T06:12:58,163 DEBUG [Listener at localhost/42843 {}] mob.MobFileCache(121): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2023-09-29T06:12:58,164 INFO [Listener at localhost/42843 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-29T06:12:58,167 INFO [Listener at localhost/42843 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-29T06:12:58,171 INFO [Listener at localhost/42843 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=regionserver:39783 connecting to ZooKeeper ensemble=127.0.0.1:49344 2023-09-29T06:12:58,177 DEBUG [Listener at localhost/42843-EventThread {}] zookeeper.ZKWatcher(604): regionserver:397830x0, quorum=127.0.0.1:49344, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-09-29T06:12:58,178 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): regionserver:39783-0x101625af8180001 connected 2023-09-29T06:12:58,178 DEBUG [Listener at localhost/42843 {}] zookeeper.ZKUtil(113): regionserver:397830x0, quorum=127.0.0.1:49344, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2023-09-29T06:12:58,180 DEBUG [Listener at localhost/42843 {}] zookeeper.ZKUtil(113): regionserver:39783-0x101625af8180001, quorum=127.0.0.1:49344, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2023-09-29T06:12:58,182 DEBUG [Listener at localhost/42843 {}] zookeeper.ZKUtil(113): regionserver:39783-0x101625af8180001, quorum=127.0.0.1:49344, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2023-09-29T06:12:58,183 DEBUG [Listener at localhost/42843 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39783 2023-09-29T06:12:58,184 DEBUG [Listener at localhost/42843 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39783 2023-09-29T06:12:58,185 DEBUG [Listener at localhost/42843 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39783 2023-09-29T06:12:58,186 DEBUG [Listener at localhost/42843 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39783 2023-09-29T06:12:58,186 DEBUG [Listener at localhost/42843 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39783 2023-09-29T06:12:58,190 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2388): Adding backup master ZNode /hbase/backup-masters/jenkins-hbase3.apache.org,42173,1695967976351 2023-09-29T06:12:58,201 DEBUG [Listener at localhost/42843-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39783-0x101625af8180001, quorum=127.0.0.1:49344, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2023-09-29T06:12:58,201 DEBUG [Listener at localhost/42843-EventThread {}] zookeeper.ZKWatcher(604): master:42173-0x101625af8180000, quorum=127.0.0.1:49344, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2023-09-29T06:12:58,205 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42173-0x101625af8180000, quorum=127.0.0.1:49344, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/jenkins-hbase3.apache.org,42173,1695967976351 2023-09-29T06:12:58,231 DEBUG [M:0;jenkins-hbase3:42173 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;jenkins-hbase3:42173 2023-09-29T06:12:58,239 DEBUG [Listener at localhost/42843-EventThread {}] zookeeper.ZKWatcher(604): master:42173-0x101625af8180000, quorum=127.0.0.1:49344, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2023-09-29T06:12:58,239 DEBUG [Listener at localhost/42843-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39783-0x101625af8180001, quorum=127.0.0.1:49344, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2023-09-29T06:12:58,239 DEBUG [Listener at localhost/42843-EventThread {}] zookeeper.ZKWatcher(604): master:42173-0x101625af8180000, quorum=127.0.0.1:49344, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-09-29T06:12:58,239 DEBUG [Listener at localhost/42843-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39783-0x101625af8180001, quorum=127.0.0.1:49344, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-09-29T06:12:58,240 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42173-0x101625af8180000, quorum=127.0.0.1:49344, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2023-09-29T06:12:58,244 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/jenkins-hbase3.apache.org,42173,1695967976351 from backup master directory 2023-09-29T06:12:58,244 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:42173-0x101625af8180000, quorum=127.0.0.1:49344, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2023-09-29T06:12:58,247 DEBUG [Listener at localhost/42843-EventThread {}] zookeeper.ZKWatcher(604): master:42173-0x101625af8180000, quorum=127.0.0.1:49344, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/jenkins-hbase3.apache.org,42173,1695967976351 2023-09-29T06:12:58,248 DEBUG [Listener at localhost/42843-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39783-0x101625af8180001, quorum=127.0.0.1:49344, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2023-09-29T06:12:58,248 DEBUG [Listener at localhost/42843-EventThread {}] zookeeper.ZKWatcher(604): master:42173-0x101625af8180000, quorum=127.0.0.1:49344, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2023-09-29T06:12:58,248 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-09-29T06:12:58,249 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=jenkins-hbase3.apache.org,42173,1695967976351 2023-09-29T06:12:58,252 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 352, initial count 0 2023-09-29T06:12:58,254 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 391, initial count 0 2023-09-29T06:12:58,756 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] util.FSUtils(611): Created cluster ID file at hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/hbase.id with ID: 3a65e529-8ef4-456d-b0f5-d9902a8f38ba 2023-09-29T06:12:58,808 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-29T06:12:58,825 DEBUG [Listener at localhost/42843-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39783-0x101625af8180001, quorum=127.0.0.1:49344, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-09-29T06:12:58,825 DEBUG [Listener at localhost/42843-EventThread {}] zookeeper.ZKWatcher(604): master:42173-0x101625af8180000, quorum=127.0.0.1:49344, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-09-29T06:12:59,275 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(355): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-29T06:12:59,277 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2023-09-29T06:12:59,296 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(271): ClientProtocol::create wrong number of arguments, should be hadoop 3.2 or below 2023-09-29T06:12:59,296 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(277): ClientProtocol::create wrong number of arguments, should be hadoop 2.x 2023-09-29T06:12:59,298 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(286): can not find SHOULD_REPLICATE flag, should be hadoop 2.x java.lang.IllegalArgumentException: No enum constant org.apache.hadoop.fs.CreateFlag.SHOULD_REPLICATE at java.lang.Enum.valueOf(Enum.java:238) ~[?:1.8.0_362] at org.apache.hadoop.fs.CreateFlag.valueOf(CreateFlag.java:63) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.loadShouldReplicateFlag(FanOutOneBlockAsyncDFSOutputHelper.java:284) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.(FanOutOneBlockAsyncDFSOutputHelper.java:311) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:140) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:951) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2413) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:574) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:571) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:12:59,307 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:141) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:951) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2413) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:574) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:571) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:12:59,309 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-09-29T06:12:59,394 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7093): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/MasterData/data/master/store 2023-09-29T06:12:59,821 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2023-09-29T06:12:59,822 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:12:59,822 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-09-29T06:12:59,822 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-29T06:12:59,823 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-29T06:12:59,823 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-09-29T06:12:59,823 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-29T06:12:59,823 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-29T06:12:59,823 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-09-29T06:12:59,826 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(232): failed to clean up initializing flag: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/MasterData/data/master/store/.initializing 2023-09-29T06:12:59,826 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(202): WALDir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/MasterData/WALs/jenkins-hbase3.apache.org,42173,1695967976351 2023-09-29T06:12:59,839 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-09-29T06:12:59,859 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C42173%2C1695967976351, suffix=, logDir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/MasterData/WALs/jenkins-hbase3.apache.org,42173,1695967976351, archiveDir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/MasterData/oldWALs, maxLogs=10 2023-09-29T06:12:59,898 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/MasterData/WALs/jenkins-hbase3.apache.org,42173,1695967976351/jenkins-hbase3.apache.org%2C42173%2C1695967976351.1695967979867, exclude list is [], retry=0 2023-09-29T06:12:59,934 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42427,DS-2e154dbb-27b5-42a6-ae9b-87cdc4287cdf,DISK] 2023-09-29T06:12:59,945 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(123): Hadoop 3.2 and below use unshaded protobuf. java.lang.ClassNotFoundException: org.apache.hadoop.thirdparty.protobuf.MessageLite at java.net.URLClassLoader.findClass(URLClassLoader.java:387) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:418) ~[?:1.8.0_362] at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:351) ~[?:1.8.0_362] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.ProtobufDecoder.(ProtobufDecoder.java:118) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.processWriteBlockResponse(FanOutOneBlockAsyncDFSOutputHelper.java:347) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$100(FanOutOneBlockAsyncDFSOutputHelper.java:121) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$4.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:431) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:185) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:35) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hadoop.hbase.util.NettyFutureUtils.addListener(NettyFutureUtils.java:52) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:425) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$300(FanOutOneBlockAsyncDFSOutputHelper.java:121) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:483) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:478) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:653) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:691) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:489) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:397) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.4.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:00,030 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/MasterData/WALs/jenkins-hbase3.apache.org,42173,1695967976351/jenkins-hbase3.apache.org%2C42173%2C1695967976351.1695967979867 2023-09-29T06:13:00,032 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:42427,DS-2e154dbb-27b5-42a6-ae9b-87cdc4287cdf,DISK]] 2023-09-29T06:13:00,033 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7254): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2023-09-29T06:13:00,033 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:13:00,039 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7296): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2023-09-29T06:13:00,040 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7299): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2023-09-29T06:13:00,110 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2023-09-29T06:13:00,146 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2023-09-29T06:13:00,151 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:13:00,155 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-29T06:13:00,155 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2023-09-29T06:13:00,160 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2023-09-29T06:13:00,160 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:13:00,161 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:13:00,162 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2023-09-29T06:13:00,164 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2023-09-29T06:13:00,165 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:13:00,165 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:13:00,166 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2023-09-29T06:13:00,180 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2023-09-29T06:13:00,180 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:13:00,181 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:13:00,186 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-09-29T06:13:00,187 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-09-29T06:13:00,204 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2023-09-29T06:13:00,209 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1076): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2023-09-29T06:13:00,214 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-29T06:13:00,215 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1093): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73575284, jitterRate=0.09635716676712036}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2023-09-29T06:13:00,224 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(992): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2023-09-29T06:13:00,226 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2023-09-29T06:13:00,277 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21cbf5eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:13:00,336 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(860): No meta location available on zookeeper, skip migrating... 2023-09-29T06:13:00,357 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2023-09-29T06:13:00,358 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(563): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2023-09-29T06:13:00,362 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2023-09-29T06:13:00,364 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(583): Recovered RegionProcedureStore lease in 1 msec 2023-09-29T06:13:00,372 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(597): Loaded RegionProcedureStore in 7 msec 2023-09-29T06:13:00,372 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2023-09-29T06:13:00,411 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2023-09-29T06:13:00,430 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42173-0x101625af8180000, quorum=127.0.0.1:49344, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2023-09-29T06:13:00,433 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /hbase/balancer already deleted, retry=false 2023-09-29T06:13:00,437 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2023-09-29T06:13:00,439 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42173-0x101625af8180000, quorum=127.0.0.1:49344, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2023-09-29T06:13:00,441 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /hbase/normalizer already deleted, retry=false 2023-09-29T06:13:00,443 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2023-09-29T06:13:00,449 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42173-0x101625af8180000, quorum=127.0.0.1:49344, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2023-09-29T06:13:00,451 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /hbase/switch/split already deleted, retry=false 2023-09-29T06:13:00,452 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42173-0x101625af8180000, quorum=127.0.0.1:49344, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2023-09-29T06:13:00,458 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /hbase/switch/merge already deleted, retry=false 2023-09-29T06:13:00,476 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42173-0x101625af8180000, quorum=127.0.0.1:49344, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2023-09-29T06:13:00,477 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /hbase/snapshot-cleanup already deleted, retry=false 2023-09-29T06:13:00,482 DEBUG [Listener at localhost/42843-EventThread {}] zookeeper.ZKWatcher(604): master:42173-0x101625af8180000, quorum=127.0.0.1:49344, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2023-09-29T06:13:00,482 DEBUG [Listener at localhost/42843-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39783-0x101625af8180001, quorum=127.0.0.1:49344, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2023-09-29T06:13:00,483 DEBUG [Listener at localhost/42843-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39783-0x101625af8180001, quorum=127.0.0.1:49344, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-09-29T06:13:00,483 DEBUG [Listener at localhost/42843-EventThread {}] zookeeper.ZKWatcher(604): master:42173-0x101625af8180000, quorum=127.0.0.1:49344, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-09-29T06:13:00,484 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(804): Active/primary master=jenkins-hbase3.apache.org,42173,1695967976351, sessionid=0x101625af8180000, setting cluster-up flag (Was=false) 2023-09-29T06:13:00,501 DEBUG [Listener at localhost/42843-EventThread {}] zookeeper.ZKWatcher(604): master:42173-0x101625af8180000, quorum=127.0.0.1:49344, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-09-29T06:13:00,501 DEBUG [Listener at localhost/42843-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39783-0x101625af8180001, quorum=127.0.0.1:49344, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-09-29T06:13:00,507 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2023-09-29T06:13:00,510 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,42173,1695967976351 2023-09-29T06:13:00,517 DEBUG [Listener at localhost/42843-EventThread {}] zookeeper.ZKWatcher(604): master:42173-0x101625af8180000, quorum=127.0.0.1:49344, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-09-29T06:13:00,517 DEBUG [Listener at localhost/42843-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39783-0x101625af8180001, quorum=127.0.0.1:49344, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-09-29T06:13:00,523 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2023-09-29T06:13:00,525 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,42173,1695967976351 2023-09-29T06:13:00,606 DEBUG [RS:0;jenkins-hbase3:39783 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;jenkins-hbase3:39783 2023-09-29T06:13:00,609 INFO [RS:0;jenkins-hbase3:39783 {}] regionserver.HRegionServer(997): ClusterId : 3a65e529-8ef4-456d-b0f5-d9902a8f38ba 2023-09-29T06:13:00,613 DEBUG [RS:0;jenkins-hbase3:39783 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2023-09-29T06:13:00,619 DEBUG [RS:0;jenkins-hbase3:39783 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2023-09-29T06:13:00,619 DEBUG [RS:0;jenkins-hbase3:39783 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2023-09-29T06:13:00,623 DEBUG [RS:0;jenkins-hbase3:39783 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2023-09-29T06:13:00,625 DEBUG [RS:0;jenkins-hbase3:39783 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@283d8e25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:13:00,632 DEBUG [RS:0;jenkins-hbase3:39783 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d1a198a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-09-29T06:13:00,639 INFO [RS:0;jenkins-hbase3:39783 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2023-09-29T06:13:00,639 INFO [RS:0;jenkins-hbase3:39783 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2023-09-29T06:13:00,639 DEBUG [RS:0;jenkins-hbase3:39783 {}] regionserver.HRegionServer(1079): About to register with Master. 2023-09-29T06:13:00,643 INFO [RS:0;jenkins-hbase3:39783 {}] regionserver.HRegionServer(3027): reportForDuty to master=jenkins-hbase3.apache.org,42173,1695967976351 with isa=jenkins-hbase3.apache.org/172.31.12.81:39783, startcode=1695967978138 2023-09-29T06:13:00,665 DEBUG [RS:0;jenkins-hbase3:39783 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-09-29T06:13:00,676 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2023-09-29T06:13:00,686 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(574): slop=0.2 2023-09-29T06:13:00,694 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(285): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2023-09-29T06:13:00,710 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: jenkins-hbase3.apache.org,42173,1695967976351 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2023-09-29T06:13:00,719 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-09-29T06:13:00,719 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-09-29T06:13:00,719 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-09-29T06:13:00,720 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-09-29T06:13:00,720 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/jenkins-hbase3:0, corePoolSize=10, maxPoolSize=10 2023-09-29T06:13:00,720 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-29T06:13:00,720 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-09-29T06:13:00,720 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-29T06:13:00,730 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1695968010729 2023-09-29T06:13:00,733 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2023-09-29T06:13:00,740 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2023-09-29T06:13:00,741 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2023-09-29T06:13:00,742 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:55009, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-09-29T06:13:00,748 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:13:00,749 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2023-09-29T06:13:00,749 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-09-29T06:13:00,754 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42173 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3181) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:590) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17393) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:00,759 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2023-09-29T06:13:00,760 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2023-09-29T06:13:00,760 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2023-09-29T06:13:00,760 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2023-09-29T06:13:00,766 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-09-29T06:13:00,770 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2023-09-29T06:13:00,774 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2023-09-29T06:13:00,775 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2023-09-29T06:13:00,783 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2023-09-29T06:13:00,783 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2023-09-29T06:13:00,787 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1695967980786,5,FailOnTimeoutGroup] 2023-09-29T06:13:00,787 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1695967980787,5,FailOnTimeoutGroup] 2023-09-29T06:13:00,788 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-09-29T06:13:00,788 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1635): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2023-09-29T06:13:00,789 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2023-09-29T06:13:00,790 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2023-09-29T06:13:00,796 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2023-09-29T06:13:00,797 INFO [PEWorker-1 {}] regionserver.HRegion(7075): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a 2023-09-29T06:13:00,803 DEBUG [RS:0;jenkins-hbase3:39783 {}] regionserver.HRegionServer(3046): Master is not running yet 2023-09-29T06:13:00,804 WARN [RS:0;jenkins-hbase3:39783 {}] regionserver.HRegionServer(1088): reportForDuty failed; sleeping 100 ms and then retrying. 2023-09-29T06:13:00,828 DEBUG [PEWorker-1 {}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:13:00,831 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-09-29T06:13:00,834 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-09-29T06:13:00,834 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:13:00,836 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-29T06:13:00,836 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-09-29T06:13:00,839 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-09-29T06:13:00,839 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:13:00,840 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-29T06:13:00,840 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-09-29T06:13:00,842 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-09-29T06:13:00,842 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:13:00,843 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-29T06:13:00,845 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/hbase/meta/1588230740 2023-09-29T06:13:00,846 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/hbase/meta/1588230740 2023-09-29T06:13:00,850 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2023-09-29T06:13:00,853 DEBUG [PEWorker-1 {}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-09-29T06:13:00,858 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-29T06:13:00,859 INFO [PEWorker-1 {}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59062519, jitterRate=-0.11989988386631012}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2023-09-29T06:13:00,861 DEBUG [PEWorker-1 {}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-09-29T06:13:00,861 DEBUG [PEWorker-1 {}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-09-29T06:13:00,861 INFO [PEWorker-1 {}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-09-29T06:13:00,862 DEBUG [PEWorker-1 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-09-29T06:13:00,862 DEBUG [PEWorker-1 {}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-09-29T06:13:00,862 DEBUG [PEWorker-1 {}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-09-29T06:13:00,867 INFO [PEWorker-1 {}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-09-29T06:13:00,867 DEBUG [PEWorker-1 {}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-09-29T06:13:00,872 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2023-09-29T06:13:00,873 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2023-09-29T06:13:00,885 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2023-09-29T06:13:00,900 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2023-09-29T06:13:00,905 INFO [RS:0;jenkins-hbase3:39783 {}] regionserver.HRegionServer(3027): reportForDuty to master=jenkins-hbase3.apache.org,42173,1695967976351 with isa=jenkins-hbase3.apache.org/172.31.12.81:39783, startcode=1695967978138 2023-09-29T06:13:00,905 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2023-09-29T06:13:00,915 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42173 {}] master.ServerManager(403): Registering regionserver=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:00,929 DEBUG [RS:0;jenkins-hbase3:39783 {}] regionserver.HRegionServer(1705): Config from master: hbase.rootdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a 2023-09-29T06:13:00,929 DEBUG [RS:0;jenkins-hbase3:39783 {}] regionserver.HRegionServer(1705): Config from master: fs.defaultFS=hdfs://localhost:34233 2023-09-29T06:13:00,929 DEBUG [RS:0;jenkins-hbase3:39783 {}] regionserver.HRegionServer(1705): Config from master: hbase.master.info.port=-1 2023-09-29T06:13:00,936 DEBUG [Listener at localhost/42843-EventThread {}] zookeeper.ZKWatcher(604): master:42173-0x101625af8180000, quorum=127.0.0.1:49344, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2023-09-29T06:13:00,937 DEBUG [RS:0;jenkins-hbase3:39783 {}] zookeeper.ZKUtil(111): regionserver:39783-0x101625af8180001, quorum=127.0.0.1:49344, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:00,937 WARN [RS:0;jenkins-hbase3:39783 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-09-29T06:13:00,937 INFO [RS:0;jenkins-hbase3:39783 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-09-29T06:13:00,937 DEBUG [RS:0;jenkins-hbase3:39783 {}] regionserver.HRegionServer(2060): logDir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/WALs/jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:00,943 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [jenkins-hbase3.apache.org,39783,1695967978138] 2023-09-29T06:13:00,959 DEBUG [RS:0;jenkins-hbase3:39783 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2023-09-29T06:13:00,975 INFO [RS:0;jenkins-hbase3:39783 {}] regionserver.MetricsRegionServerWrapperImpl(165): Computing regionserver metrics every 5000 milliseconds 2023-09-29T06:13:00,992 INFO [RS:0;jenkins-hbase3:39783 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=782.4 M, globalMemStoreLimitLowMark=743.3 M, Offheap=false 2023-09-29T06:13:00,997 INFO [RS:0;jenkins-hbase3:39783 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2023-09-29T06:13:00,998 INFO [RS:0;jenkins-hbase3:39783 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2023-09-29T06:13:01,002 INFO [RS:0;jenkins-hbase3:39783 {}] regionserver.HRegionServer$CompactionChecker(1947): CompactionChecker runs every PT1S 2023-09-29T06:13:01,013 INFO [RS:0;jenkins-hbase3:39783 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2023-09-29T06:13:01,013 DEBUG [RS:0;jenkins-hbase3:39783 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-29T06:13:01,014 DEBUG [RS:0;jenkins-hbase3:39783 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-29T06:13:01,014 DEBUG [RS:0;jenkins-hbase3:39783 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-29T06:13:01,014 DEBUG [RS:0;jenkins-hbase3:39783 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-29T06:13:01,014 DEBUG [RS:0;jenkins-hbase3:39783 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-29T06:13:01,014 DEBUG [RS:0;jenkins-hbase3:39783 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-09-29T06:13:01,015 DEBUG [RS:0;jenkins-hbase3:39783 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-29T06:13:01,015 DEBUG [RS:0;jenkins-hbase3:39783 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-29T06:13:01,015 DEBUG [RS:0;jenkins-hbase3:39783 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-29T06:13:01,015 DEBUG [RS:0;jenkins-hbase3:39783 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-29T06:13:01,015 DEBUG [RS:0;jenkins-hbase3:39783 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-29T06:13:01,016 DEBUG [RS:0;jenkins-hbase3:39783 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2023-09-29T06:13:01,016 DEBUG [RS:0;jenkins-hbase3:39783 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2023-09-29T06:13:01,017 INFO [RS:0;jenkins-hbase3:39783 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2023-09-29T06:13:01,017 INFO [RS:0;jenkins-hbase3:39783 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2023-09-29T06:13:01,017 INFO [RS:0;jenkins-hbase3:39783 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2023-09-29T06:13:01,017 INFO [RS:0;jenkins-hbase3:39783 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2023-09-29T06:13:01,017 INFO [RS:0;jenkins-hbase3:39783 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,39783,1695967978138-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-09-29T06:13:01,035 INFO [RS:0;jenkins-hbase3:39783 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2023-09-29T06:13:01,039 INFO [RS:0;jenkins-hbase3:39783 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,39783,1695967978138-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2023-09-29T06:13:01,057 WARN [jenkins-hbase3:42173 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-29T06:13:01,060 INFO [RS:0;jenkins-hbase3:39783 {}] regionserver.Replication(204): jenkins-hbase3.apache.org,39783,1695967978138 started 2023-09-29T06:13:01,060 INFO [RS:0;jenkins-hbase3:39783 {}] regionserver.HRegionServer(1747): Serving as jenkins-hbase3.apache.org,39783,1695967978138, RpcServer on jenkins-hbase3.apache.org/172.31.12.81:39783, sessionid=0x101625af8180001 2023-09-29T06:13:01,061 DEBUG [RS:0;jenkins-hbase3:39783 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2023-09-29T06:13:01,061 DEBUG [RS:0;jenkins-hbase3:39783 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:01,061 DEBUG [RS:0;jenkins-hbase3:39783 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,39783,1695967978138' 2023-09-29T06:13:01,062 DEBUG [RS:0;jenkins-hbase3:39783 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2023-09-29T06:13:01,063 DEBUG [RS:0;jenkins-hbase3:39783 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2023-09-29T06:13:01,063 DEBUG [RS:0;jenkins-hbase3:39783 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2023-09-29T06:13:01,064 DEBUG [RS:0;jenkins-hbase3:39783 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2023-09-29T06:13:01,064 DEBUG [RS:0;jenkins-hbase3:39783 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:01,064 DEBUG [RS:0;jenkins-hbase3:39783 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,39783,1695967978138' 2023-09-29T06:13:01,064 DEBUG [RS:0;jenkins-hbase3:39783 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2023-09-29T06:13:01,067 DEBUG [RS:0;jenkins-hbase3:39783 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2023-09-29T06:13:01,068 DEBUG [RS:0;jenkins-hbase3:39783 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2023-09-29T06:13:01,068 INFO [RS:0;jenkins-hbase3:39783 {}] quotas.RegionServerRpcQuotaManager(63): Quota support disabled 2023-09-29T06:13:01,068 INFO [RS:0;jenkins-hbase3:39783 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2023-09-29T06:13:01,180 INFO [RS:0;jenkins-hbase3:39783 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-09-29T06:13:01,183 INFO [RS:0;jenkins-hbase3:39783 {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C39783%2C1695967978138, suffix=, logDir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/WALs/jenkins-hbase3.apache.org,39783,1695967978138, archiveDir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/oldWALs, maxLogs=32 2023-09-29T06:13:01,197 DEBUG [RS:0;jenkins-hbase3:39783 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/WALs/jenkins-hbase3.apache.org,39783,1695967978138/jenkins-hbase3.apache.org%2C39783%2C1695967978138.1695967981185, exclude list is [], retry=0 2023-09-29T06:13:01,205 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42427,DS-2e154dbb-27b5-42a6-ae9b-87cdc4287cdf,DISK] 2023-09-29T06:13:01,210 INFO [RS:0;jenkins-hbase3:39783 {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/WALs/jenkins-hbase3.apache.org,39783,1695967978138/jenkins-hbase3.apache.org%2C39783%2C1695967978138.1695967981185 2023-09-29T06:13:01,211 DEBUG [RS:0;jenkins-hbase3:39783 {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:42427,DS-2e154dbb-27b5-42a6-ae9b-87cdc4287cdf,DISK]] 2023-09-29T06:13:01,309 DEBUG [jenkins-hbase3:42173 {}] assignment.AssignmentManager(2303): Processing assignQueue; systemServersCount=1, allServersCount=1 2023-09-29T06:13:01,317 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:01,324 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,39783,1695967978138, state=OPENING 2023-09-29T06:13:01,334 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2023-09-29T06:13:01,335 DEBUG [Listener at localhost/42843-EventThread {}] zookeeper.ZKWatcher(604): master:42173-0x101625af8180000, quorum=127.0.0.1:49344, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-09-29T06:13:01,335 DEBUG [Listener at localhost/42843-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39783-0x101625af8180001, quorum=127.0.0.1:49344, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-09-29T06:13:01,336 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2023-09-29T06:13:01,336 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2023-09-29T06:13:01,339 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,39783,1695967978138}] 2023-09-29T06:13:01,531 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(721): New admin connection to jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:01,534 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=AdminService, sasl=false 2023-09-29T06:13:01,538 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:54680, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2023-09-29T06:13:01,552 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(130): Open hbase:meta,,1.1588230740 2023-09-29T06:13:01,553 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-09-29T06:13:01,553 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2023-09-29T06:13:01,557 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C39783%2C1695967978138.meta, suffix=.meta, logDir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/WALs/jenkins-hbase3.apache.org,39783,1695967978138, archiveDir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/oldWALs, maxLogs=32 2023-09-29T06:13:01,574 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/WALs/jenkins-hbase3.apache.org,39783,1695967978138/jenkins-hbase3.apache.org%2C39783%2C1695967978138.meta.1695967981560.meta, exclude list is [], retry=0 2023-09-29T06:13:01,582 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42427,DS-2e154dbb-27b5-42a6-ae9b-87cdc4287cdf,DISK] 2023-09-29T06:13:01,593 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/WALs/jenkins-hbase3.apache.org,39783,1695967978138/jenkins-hbase3.apache.org%2C39783%2C1695967978138.meta.1695967981560.meta 2023-09-29T06:13:01,593 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:42427,DS-2e154dbb-27b5-42a6-ae9b-87cdc4287cdf,DISK]] 2023-09-29T06:13:01,593 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7254): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2023-09-29T06:13:01,596 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2023-09-29T06:13:01,617 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7955): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2023-09-29T06:13:01,623 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(393): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2023-09-29T06:13:01,629 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2023-09-29T06:13:01,630 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:13:01,630 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7296): checking encryption for 1588230740 2023-09-29T06:13:01,630 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7299): checking classloading for 1588230740 2023-09-29T06:13:01,634 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-09-29T06:13:01,635 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-09-29T06:13:01,636 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:13:01,637 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-29T06:13:01,637 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-09-29T06:13:01,639 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-09-29T06:13:01,639 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:13:01,640 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-29T06:13:01,640 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-09-29T06:13:01,642 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-09-29T06:13:01,642 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:13:01,643 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-29T06:13:01,645 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/hbase/meta/1588230740 2023-09-29T06:13:01,648 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/hbase/meta/1588230740 2023-09-29T06:13:01,651 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2023-09-29T06:13:01,654 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-09-29T06:13:01,656 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60469770, jitterRate=-0.09893020987510681}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2023-09-29T06:13:01,657 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-09-29T06:13:01,671 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1695967981522 2023-09-29T06:13:01,693 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:meta,,1.1588230740 2023-09-29T06:13:01,694 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(158): Opened hbase:meta,,1.1588230740 2023-09-29T06:13:01,695 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:01,698 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,39783,1695967978138, state=OPEN 2023-09-29T06:13:01,701 DEBUG [Listener at localhost/42843-EventThread {}] zookeeper.ZKWatcher(604): master:42173-0x101625af8180000, quorum=127.0.0.1:49344, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2023-09-29T06:13:01,701 DEBUG [Listener at localhost/42843-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39783-0x101625af8180001, quorum=127.0.0.1:49344, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2023-09-29T06:13:01,701 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2023-09-29T06:13:01,701 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2023-09-29T06:13:01,705 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=3, resume processing ppid=2 2023-09-29T06:13:01,706 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,39783,1695967978138 in 362 msec 2023-09-29T06:13:01,718 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=2, resume processing ppid=1 2023-09-29T06:13:01,718 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 823 msec 2023-09-29T06:13:01,724 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 1.1370 sec 2023-09-29T06:13:01,724 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1063): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1695967981724, completionTime=-1 2023-09-29T06:13:01,724 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ServerManager(830): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2023-09-29T06:13:01,725 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1629): Joining cluster... 2023-09-29T06:13:01,798 DEBUG [hconnection-0x12d7eb4f-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:13:01,802 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:54694, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:13:01,816 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1641): Number of RegionServers=1 2023-09-29T06:13:01,817 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1695968041816 2023-09-29T06:13:01,817 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1695968101817 2023-09-29T06:13:01,817 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1648): Joined the cluster in 92 msec 2023-09-29T06:13:01,854 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,42173,1695967976351-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2023-09-29T06:13:01,855 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,42173,1695967976351-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2023-09-29T06:13:01,855 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,42173,1695967976351-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2023-09-29T06:13:01,857 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-jenkins-hbase3:42173, period=300000, unit=MILLISECONDS is enabled. 2023-09-29T06:13:01,858 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2023-09-29T06:13:01,865 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] janitor.CatalogJanitor(175): 2023-09-29T06:13:01,878 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2023-09-29T06:13:01,880 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2368): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-09-29T06:13:01,890 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2023-09-29T06:13:01,895 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2023-09-29T06:13:01,897 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:13:01,899 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-29T06:13:01,920 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 013609be28cb78869a9efb7623dcb4f7, NAME => 'hbase:namespace,,1695967981879.013609be28cb78869a9efb7623dcb4f7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a 2023-09-29T06:13:01,935 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:namespace,,1695967981879.013609be28cb78869a9efb7623dcb4f7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:13:01,935 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1668): Closing 013609be28cb78869a9efb7623dcb4f7, disabling compactions & flushes 2023-09-29T06:13:01,935 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:namespace,,1695967981879.013609be28cb78869a9efb7623dcb4f7. 2023-09-29T06:13:01,935 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1695967981879.013609be28cb78869a9efb7623dcb4f7. 2023-09-29T06:13:01,935 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1695967981879.013609be28cb78869a9efb7623dcb4f7. after waiting 0 ms 2023-09-29T06:13:01,935 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1695967981879.013609be28cb78869a9efb7623dcb4f7. 2023-09-29T06:13:01,935 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1909): Closed hbase:namespace,,1695967981879.013609be28cb78869a9efb7623dcb4f7. 2023-09-29T06:13:01,935 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1622): Region close journal for 013609be28cb78869a9efb7623dcb4f7: 2023-09-29T06:13:01,938 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2023-09-29T06:13:01,950 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1695967981879.013609be28cb78869a9efb7623dcb4f7.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1695967981939"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1695967981939"}]},"ts":"1695967981939"} 2023-09-29T06:13:01,984 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-29T06:13:01,986 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-29T06:13:01,992 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695967981986"}]},"ts":"1695967981986"} 2023-09-29T06:13:01,997 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2023-09-29T06:13:02,004 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=013609be28cb78869a9efb7623dcb4f7, ASSIGN}] 2023-09-29T06:13:02,007 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=013609be28cb78869a9efb7623dcb4f7, ASSIGN 2023-09-29T06:13:02,009 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=013609be28cb78869a9efb7623dcb4f7, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,39783,1695967978138; forceNewPlan=false, retain=false 2023-09-29T06:13:02,160 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=013609be28cb78869a9efb7623dcb4f7, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:02,166 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 013609be28cb78869a9efb7623dcb4f7, server=jenkins-hbase3.apache.org,39783,1695967978138}] 2023-09-29T06:13:02,326 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(130): Open hbase:namespace,,1695967981879.013609be28cb78869a9efb7623dcb4f7. 2023-09-29T06:13:02,327 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7254): Opening region: {ENCODED => 013609be28cb78869a9efb7623dcb4f7, NAME => 'hbase:namespace,,1695967981879.013609be28cb78869a9efb7623dcb4f7.', STARTKEY => '', ENDKEY => ''} 2023-09-29T06:13:02,328 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 013609be28cb78869a9efb7623dcb4f7 2023-09-29T06:13:02,328 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(888): Instantiated hbase:namespace,,1695967981879.013609be28cb78869a9efb7623dcb4f7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:13:02,328 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7296): checking encryption for 013609be28cb78869a9efb7623dcb4f7 2023-09-29T06:13:02,328 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7299): checking classloading for 013609be28cb78869a9efb7623dcb4f7 2023-09-29T06:13:02,330 INFO [StoreOpener-013609be28cb78869a9efb7623dcb4f7-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 013609be28cb78869a9efb7623dcb4f7 2023-09-29T06:13:02,333 INFO [StoreOpener-013609be28cb78869a9efb7623dcb4f7-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 013609be28cb78869a9efb7623dcb4f7 columnFamilyName info 2023-09-29T06:13:02,333 DEBUG [StoreOpener-013609be28cb78869a9efb7623dcb4f7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:13:02,334 INFO [StoreOpener-013609be28cb78869a9efb7623dcb4f7-1 {}] regionserver.HStore(324): Store=013609be28cb78869a9efb7623dcb4f7/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:13:02,335 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/hbase/namespace/013609be28cb78869a9efb7623dcb4f7 2023-09-29T06:13:02,336 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/hbase/namespace/013609be28cb78869a9efb7623dcb4f7 2023-09-29T06:13:02,340 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1076): writing seq id for 013609be28cb78869a9efb7623dcb4f7 2023-09-29T06:13:02,348 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/hbase/namespace/013609be28cb78869a9efb7623dcb4f7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-29T06:13:02,349 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1093): Opened 013609be28cb78869a9efb7623dcb4f7; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62279950, jitterRate=-0.07195642590522766}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-29T06:13:02,350 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(992): Region open journal for 013609be28cb78869a9efb7623dcb4f7: 2023-09-29T06:13:02,353 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:namespace,,1695967981879.013609be28cb78869a9efb7623dcb4f7., pid=6, masterSystemTime=1695967982319 2023-09-29T06:13:02,358 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:namespace,,1695967981879.013609be28cb78869a9efb7623dcb4f7. 2023-09-29T06:13:02,358 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(158): Opened hbase:namespace,,1695967981879.013609be28cb78869a9efb7623dcb4f7. 2023-09-29T06:13:02,359 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=013609be28cb78869a9efb7623dcb4f7, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:02,367 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=6, resume processing ppid=5 2023-09-29T06:13:02,367 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 013609be28cb78869a9efb7623dcb4f7, server=jenkins-hbase3.apache.org,39783,1695967978138 in 198 msec 2023-09-29T06:13:02,372 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=5, resume processing ppid=4 2023-09-29T06:13:02,372 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=013609be28cb78869a9efb7623dcb4f7, ASSIGN in 363 msec 2023-09-29T06:13:02,373 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-29T06:13:02,374 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695967982374"}]},"ts":"1695967982374"} 2023-09-29T06:13:02,377 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2023-09-29T06:13:02,382 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2023-09-29T06:13:02,385 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 501 msec 2023-09-29T06:13:02,395 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:42173-0x101625af8180000, quorum=127.0.0.1:49344, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace 2023-09-29T06:13:02,396 DEBUG [Listener at localhost/42843-EventThread {}] zookeeper.ZKWatcher(604): master:42173-0x101625af8180000, quorum=127.0.0.1:49344, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace 2023-09-29T06:13:02,396 DEBUG [Listener at localhost/42843-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39783-0x101625af8180001, quorum=127.0.0.1:49344, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-09-29T06:13:02,396 DEBUG [Listener at localhost/42843-EventThread {}] zookeeper.ZKWatcher(604): master:42173-0x101625af8180000, quorum=127.0.0.1:49344, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-09-29T06:13:02,453 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2023-09-29T06:13:02,478 DEBUG [Listener at localhost/42843-EventThread {}] zookeeper.ZKWatcher(604): master:42173-0x101625af8180000, quorum=127.0.0.1:49344, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2023-09-29T06:13:02,485 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 42 msec 2023-09-29T06:13:02,500 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2023-09-29T06:13:02,515 DEBUG [Listener at localhost/42843-EventThread {}] zookeeper.ZKWatcher(604): master:42173-0x101625af8180000, quorum=127.0.0.1:49344, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2023-09-29T06:13:02,524 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 23 msec 2023-09-29T06:13:02,539 DEBUG [Listener at localhost/42843-EventThread {}] zookeeper.ZKWatcher(604): master:42173-0x101625af8180000, quorum=127.0.0.1:49344, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default 2023-09-29T06:13:02,543 DEBUG [Listener at localhost/42843-EventThread {}] zookeeper.ZKWatcher(604): master:42173-0x101625af8180000, quorum=127.0.0.1:49344, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase 2023-09-29T06:13:02,543 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1193): Master has completed initialization 4.294sec 2023-09-29T06:13:02,545 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2023-09-29T06:13:02,547 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2023-09-29T06:13:02,549 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2023-09-29T06:13:02,550 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2023-09-29T06:13:02,550 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2023-09-29T06:13:02,553 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,42173,1695967976351-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-09-29T06:13:02,553 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,42173,1695967976351-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2023-09-29T06:13:02,568 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1296): Balancer post startup initialization complete, took 0 seconds 2023-09-29T06:13:02,569 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2023-09-29T06:13:02,615 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x3f46555d to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:13:02,629 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7abdd0b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:13:02,635 DEBUG [Listener at localhost/42843 {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2023-09-29T06:13:02,635 DEBUG [Listener at localhost/42843 {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2023-09-29T06:13:02,652 DEBUG [hconnection-0x3f481cd7-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:13:02,670 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:54698, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:13:02,682 INFO [Listener at localhost/42843 {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=jenkins-hbase3.apache.org,42173,1695967976351 2023-09-29T06:13:02,709 INFO [Listener at localhost/42843 {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithAdaptivePolicy#testMixedAtomicity Thread=204, OpenFileDescriptor=478, MaxFileDescriptor=60000, SystemLoadAverage=581, ProcessCount=171, AvailableMemoryMB=2568 2023-09-29T06:13:02,728 DEBUG [Listener at localhost/42843 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-29T06:13:02,738 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:34302, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-29T06:13:02,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] util.TableDescriptorChecker(342): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2023-09-29T06:13:02,758 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'ADAPTIVE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-29T06:13:02,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestAcidGuarantees 2023-09-29T06:13:02,763 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2023-09-29T06:13:02,763 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 9 2023-09-29T06:13:02,764 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:13:02,767 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=9, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-29T06:13:02,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=9 2023-09-29T06:13:02,792 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'ADAPTIVE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a 2023-09-29T06:13:02,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=9 2023-09-29T06:13:03,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=9 2023-09-29T06:13:03,208 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(888): Instantiated TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:13:03,208 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1668): Closing 1291e331f06575ebce0d38fd8ca0d9de, disabling compactions & flushes 2023-09-29T06:13:03,208 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1690): Closing region TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:03,208 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:03,209 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. after waiting 0 ms 2023-09-29T06:13:03,209 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:03,209 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1909): Closed TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:03,209 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1622): Region close journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:03,211 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=9, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2023-09-29T06:13:03,212 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1695967983212"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1695967983212"}]},"ts":"1695967983212"} 2023-09-29T06:13:03,216 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-29T06:13:03,218 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=9, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-29T06:13:03,218 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695967983218"}]},"ts":"1695967983218"} 2023-09-29T06:13:03,221 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2023-09-29T06:13:03,228 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=1291e331f06575ebce0d38fd8ca0d9de, ASSIGN}] 2023-09-29T06:13:03,231 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=1291e331f06575ebce0d38fd8ca0d9de, ASSIGN 2023-09-29T06:13:03,232 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1291e331f06575ebce0d38fd8ca0d9de, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,39783,1695967978138; forceNewPlan=false, retain=false 2023-09-29T06:13:03,384 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=1291e331f06575ebce0d38fd8ca0d9de, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:03,389 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE; OpenRegionProcedure 1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138}] 2023-09-29T06:13:03,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=9 2023-09-29T06:13:03,552 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(130): Open TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:03,552 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7254): Opening region: {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} 2023-09-29T06:13:03,553 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1291e331f06575ebce0d38fd8ca0d9de 2023-09-29T06:13:03,553 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(888): Instantiated TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:13:03,553 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7296): checking encryption for 1291e331f06575ebce0d38fd8ca0d9de 2023-09-29T06:13:03,553 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7299): checking classloading for 1291e331f06575ebce0d38fd8ca0d9de 2023-09-29T06:13:03,558 INFO [StoreOpener-1291e331f06575ebce0d38fd8ca0d9de-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1291e331f06575ebce0d38fd8ca0d9de 2023-09-29T06:13:03,563 INFO [StoreOpener-1291e331f06575ebce0d38fd8ca0d9de-1 {}] regionserver.CompactingMemStore(121): Store=A, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2023-09-29T06:13:03,563 INFO [StoreOpener-1291e331f06575ebce0d38fd8ca0d9de-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1291e331f06575ebce0d38fd8ca0d9de columnFamilyName A 2023-09-29T06:13:03,563 DEBUG [StoreOpener-1291e331f06575ebce0d38fd8ca0d9de-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:13:03,565 INFO [StoreOpener-1291e331f06575ebce0d38fd8ca0d9de-1 {}] regionserver.HStore(324): Store=1291e331f06575ebce0d38fd8ca0d9de/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:13:03,565 INFO [StoreOpener-1291e331f06575ebce0d38fd8ca0d9de-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1291e331f06575ebce0d38fd8ca0d9de 2023-09-29T06:13:03,567 INFO [StoreOpener-1291e331f06575ebce0d38fd8ca0d9de-1 {}] regionserver.CompactingMemStore(121): Store=B, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2023-09-29T06:13:03,568 INFO [StoreOpener-1291e331f06575ebce0d38fd8ca0d9de-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1291e331f06575ebce0d38fd8ca0d9de columnFamilyName B 2023-09-29T06:13:03,568 DEBUG [StoreOpener-1291e331f06575ebce0d38fd8ca0d9de-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:13:03,569 INFO [StoreOpener-1291e331f06575ebce0d38fd8ca0d9de-1 {}] regionserver.HStore(324): Store=1291e331f06575ebce0d38fd8ca0d9de/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:13:03,569 INFO [StoreOpener-1291e331f06575ebce0d38fd8ca0d9de-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1291e331f06575ebce0d38fd8ca0d9de 2023-09-29T06:13:03,571 INFO [StoreOpener-1291e331f06575ebce0d38fd8ca0d9de-1 {}] regionserver.CompactingMemStore(121): Store=C, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2023-09-29T06:13:03,572 INFO [StoreOpener-1291e331f06575ebce0d38fd8ca0d9de-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1291e331f06575ebce0d38fd8ca0d9de columnFamilyName C 2023-09-29T06:13:03,572 DEBUG [StoreOpener-1291e331f06575ebce0d38fd8ca0d9de-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:13:03,573 INFO [StoreOpener-1291e331f06575ebce0d38fd8ca0d9de-1 {}] regionserver.HStore(324): Store=1291e331f06575ebce0d38fd8ca0d9de/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:13:03,573 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1169): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:03,576 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de 2023-09-29T06:13:03,577 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de 2023-09-29T06:13:03,580 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2023-09-29T06:13:03,583 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1076): writing seq id for 1291e331f06575ebce0d38fd8ca0d9de 2023-09-29T06:13:03,588 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-29T06:13:03,589 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1093): Opened 1291e331f06575ebce0d38fd8ca0d9de; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60839261, jitterRate=-0.09342436492443085}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2023-09-29T06:13:03,591 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(992): Region open journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:03,592 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2556): Post open deploy tasks for TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de., pid=11, masterSystemTime=1695967983543 2023-09-29T06:13:03,595 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2583): Finished post open deploy task for TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:03,596 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(158): Opened TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:03,596 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=1291e331f06575ebce0d38fd8ca0d9de, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:03,603 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=11, resume processing ppid=10 2023-09-29T06:13:03,603 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=11, ppid=10, state=SUCCESS; OpenRegionProcedure 1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 in 210 msec 2023-09-29T06:13:03,607 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=10, resume processing ppid=9 2023-09-29T06:13:03,607 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=10, ppid=9, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=1291e331f06575ebce0d38fd8ca0d9de, ASSIGN in 376 msec 2023-09-29T06:13:03,609 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=9, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-29T06:13:03,609 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695967983609"}]},"ts":"1695967983609"} 2023-09-29T06:13:03,611 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2023-09-29T06:13:03,615 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=9, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2023-09-29T06:13:03,618 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=9, state=SUCCESS; CreateTableProcedure table=TestAcidGuarantees in 857 msec 2023-09-29T06:13:03,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=9 2023-09-29T06:13:03,893 INFO [Listener at localhost/42843 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: default:TestAcidGuarantees, procId: 9 completed 2023-09-29T06:13:03,900 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x19f53530 to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:13:03,906 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cd563a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:13:03,908 DEBUG [Listener at localhost/42843 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:13:03,913 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:54708, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:13:03,920 DEBUG [Listener at localhost/42843 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-29T06:13:03,923 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:34306, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-29T06:13:03,933 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x2ee95d5f to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:13:03,939 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34abc55b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:13:03,940 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x04ea4d93 to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:13:03,949 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a667ba3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:13:03,951 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x24625a3b to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:13:03,955 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bbfd708, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:13:03,957 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x70544c9c to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:13:03,963 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1303fb7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:13:03,965 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x3bf36245 to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:13:04,007 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7676360f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:13:04,009 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x171d09b4 to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:13:04,018 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7314225b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:13:04,020 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x3820db30 to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:13:04,026 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@478f2529, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:13:04,029 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x3dfed535 to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:13:04,041 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e619ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:13:04,044 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0035664b to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:13:04,049 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ba95c05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:13:04,066 DEBUG [hconnection-0x36b68f03-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:13:04,067 DEBUG [hconnection-0x3cc660fe-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:13:04,068 DEBUG [hconnection-0x4b2867b5-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:13:04,069 DEBUG [hconnection-0x5c8ae938-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:13:04,069 DEBUG [hconnection-0x3c1437a7-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:13:04,075 DEBUG [hconnection-0x1bfff401-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:13:04,076 DEBUG [hconnection-0x35edc90d-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:13:04,083 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:54710, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:13:04,086 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:54718, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:13:04,087 DEBUG [hconnection-0x2869fcf4-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:13:04,088 DEBUG [hconnection-0x1884acd-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:13:04,090 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:13:04,094 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:54730, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:13:04,099 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:54744, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:13:04,108 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:54760, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:13:04,115 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:54764, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:13:04,117 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:54756, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:13:04,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=12, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=12, table=TestAcidGuarantees 2023-09-29T06:13:04,124 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:54774, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:13:04,125 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:54780, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:13:04,125 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=12, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=12, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:13:04,130 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=12, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=12, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:13:04,134 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=13, ppid=12, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:13:04,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=12 2023-09-29T06:13:04,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 1291e331f06575ebce0d38fd8ca0d9de 2023-09-29T06:13:04,231 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 1291e331f06575ebce0d38fd8ca0d9de 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2023-09-29T06:13:04,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=12 2023-09-29T06:13:04,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=A 2023-09-29T06:13:04,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:04,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=B 2023-09-29T06:13:04,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:04,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=C 2023-09-29T06:13:04,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:04,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=13 2023-09-29T06:13:04,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:04,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. as already flushing 2023-09-29T06:13:04,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:04,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] handler.RSProcedureHandler(53): pid=13 java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:04,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=13 java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:04,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:04,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54718 deadline: 1695968044359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:04,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:04,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968044361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:04,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:04,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:04,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968044366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:04,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:04,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968044371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:04,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968044361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:04,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=13 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:04,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/f3f958725f1740dab741720d3eb189ad is 50, key is test_row_0/A:col10/1695967984221/Put/seqid=0 2023-09-29T06:13:04,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=12 2023-09-29T06:13:04,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:04,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54718 deadline: 1695968044473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:04,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:04,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968044474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:04,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:04,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968044494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:04,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:04,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968044495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:04,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:04,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968044495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:04,552 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/f3f958725f1740dab741720d3eb189ad 2023-09-29T06:13:04,562 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=13 2023-09-29T06:13:04,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:04,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. as already flushing 2023-09-29T06:13:04,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:04,574 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] handler.RSProcedureHandler(53): pid=13 java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:04,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=13 java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:04,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=13 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:04,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:04,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968044696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:04,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/5a1257a4902e435099f81cce4f0c7df9 is 50, key is test_row_0/B:col10/1695967984221/Put/seqid=0 2023-09-29T06:13:04,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:04,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54718 deadline: 1695968044698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:04,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:04,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968044706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:04,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:04,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968044706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:04,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:04,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968044706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:04,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=13 2023-09-29T06:13:04,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=12 2023-09-29T06:13:04,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:04,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. as already flushing 2023-09-29T06:13:04,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:04,779 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] handler.RSProcedureHandler(53): pid=13 java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:04,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=13 java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:04,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=13 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:04,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=13 2023-09-29T06:13:04,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:04,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. as already flushing 2023-09-29T06:13:04,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:04,981 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] handler.RSProcedureHandler(53): pid=13 java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:04,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=13 java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:04,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=13 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:05,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:05,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968045001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:05,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:05,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54718 deadline: 1695968045016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:05,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:05,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968045019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:05,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:05,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968045022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:05,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:05,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968045024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:05,141 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=13 2023-09-29T06:13:05,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:05,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. as already flushing 2023-09-29T06:13:05,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:05,146 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] handler.RSProcedureHandler(53): pid=13 java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:05,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=13 java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:05,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=13 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:05,190 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/5a1257a4902e435099f81cce4f0c7df9 2023-09-29T06:13:05,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=12 2023-09-29T06:13:05,267 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/ac24b6cd9df64fd9afa0d4bb1336af20 is 50, key is test_row_0/C:col10/1695967984221/Put/seqid=0 2023-09-29T06:13:05,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=13 2023-09-29T06:13:05,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:05,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. as already flushing 2023-09-29T06:13:05,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:05,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] handler.RSProcedureHandler(53): pid=13 java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:05,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=13 java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:05,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=13 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:05,320 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/ac24b6cd9df64fd9afa0d4bb1336af20 2023-09-29T06:13:05,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/f3f958725f1740dab741720d3eb189ad as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/f3f958725f1740dab741720d3eb189ad 2023-09-29T06:13:05,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/f3f958725f1740dab741720d3eb189ad, entries=150, sequenceid=14, filesize=11.7 K 2023-09-29T06:13:05,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/5a1257a4902e435099f81cce4f0c7df9 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/5a1257a4902e435099f81cce4f0c7df9 2023-09-29T06:13:05,384 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/5a1257a4902e435099f81cce4f0c7df9, entries=150, sequenceid=14, filesize=11.7 K 2023-09-29T06:13:05,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/ac24b6cd9df64fd9afa0d4bb1336af20 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/ac24b6cd9df64fd9afa0d4bb1336af20 2023-09-29T06:13:05,407 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/ac24b6cd9df64fd9afa0d4bb1336af20, entries=150, sequenceid=14, filesize=11.7 K 2023-09-29T06:13:05,415 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 1291e331f06575ebce0d38fd8ca0d9de in 1183ms, sequenceid=14, compaction requested=false 2023-09-29T06:13:05,417 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2023-09-29T06:13:05,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:05,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=13 2023-09-29T06:13:05,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:05,470 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(2819): Flushing 1291e331f06575ebce0d38fd8ca0d9de 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2023-09-29T06:13:05,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=A 2023-09-29T06:13:05,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:05,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=B 2023-09-29T06:13:05,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:05,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=C 2023-09-29T06:13:05,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:05,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/5f6ff41a29b549c8b74e3642fe2fbe3f is 50, key is test_row_0/A:col10/1695967984342/Put/seqid=0 2023-09-29T06:13:05,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. as already flushing 2023-09-29T06:13:05,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 1291e331f06575ebce0d38fd8ca0d9de 2023-09-29T06:13:05,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:05,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968045540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:05,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:05,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968045543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:05,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:05,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968045544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:05,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:05,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54718 deadline: 1695968045554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:05,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:05,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968045561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:05,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:05,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968045647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:05,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:05,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968045658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:05,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:05,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968045660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:05,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:05,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54718 deadline: 1695968045660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:05,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:05,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968045690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:05,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:05,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968045859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:05,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:05,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968045869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:05,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:05,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968045871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:05,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:05,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54718 deadline: 1695968045880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:05,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:05,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968045912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:05,987 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/5f6ff41a29b549c8b74e3642fe2fbe3f 2023-09-29T06:13:06,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/2304f50233a444ccbb60f3b2de131c6f is 50, key is test_row_0/B:col10/1695967984342/Put/seqid=0 2023-09-29T06:13:06,065 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/2304f50233a444ccbb60f3b2de131c6f 2023-09-29T06:13:06,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/f6782c14c6df4083a91bb4f738ca167a is 50, key is test_row_0/C:col10/1695967984342/Put/seqid=0 2023-09-29T06:13:06,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:06,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968046169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:06,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:06,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968046173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:06,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:06,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968046178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:06,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:06,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54718 deadline: 1695968046190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:06,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:06,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968046220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:06,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=12 2023-09-29T06:13:06,546 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/f6782c14c6df4083a91bb4f738ca167a 2023-09-29T06:13:06,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/5f6ff41a29b549c8b74e3642fe2fbe3f as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/5f6ff41a29b549c8b74e3642fe2fbe3f 2023-09-29T06:13:06,571 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/5f6ff41a29b549c8b74e3642fe2fbe3f, entries=150, sequenceid=37, filesize=11.7 K 2023-09-29T06:13:06,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/2304f50233a444ccbb60f3b2de131c6f as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/2304f50233a444ccbb60f3b2de131c6f 2023-09-29T06:13:06,588 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/2304f50233a444ccbb60f3b2de131c6f, entries=150, sequenceid=37, filesize=11.7 K 2023-09-29T06:13:06,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/f6782c14c6df4083a91bb4f738ca167a as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/f6782c14c6df4083a91bb4f738ca167a 2023-09-29T06:13:06,623 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/f6782c14c6df4083a91bb4f738ca167a, entries=150, sequenceid=37, filesize=11.7 K 2023-09-29T06:13:06,627 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(3022): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=80.51 KB/82440 for 1291e331f06575ebce0d38fd8ca0d9de in 1158ms, sequenceid=37, compaction requested=false 2023-09-29T06:13:06,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(2520): Flush status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:06,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:06,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=13 2023-09-29T06:13:06,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=13 2023-09-29T06:13:06,645 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=13, resume processing ppid=12 2023-09-29T06:13:06,645 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=13, ppid=12, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5080 sec 2023-09-29T06:13:06,663 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=12, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=12, table=TestAcidGuarantees in 2.5550 sec 2023-09-29T06:13:06,681 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 1291e331f06575ebce0d38fd8ca0d9de 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2023-09-29T06:13:06,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 1291e331f06575ebce0d38fd8ca0d9de 2023-09-29T06:13:06,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=A 2023-09-29T06:13:06,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:06,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=B 2023-09-29T06:13:06,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:06,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=C 2023-09-29T06:13:06,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:06,762 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/935432a161704c2b828ac0509a6523c2 is 50, key is test_row_0/A:col10/1695967986678/Put/seqid=0 2023-09-29T06:13:06,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:06,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968046794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:06,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:06,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968046814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:06,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:06,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968046815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:06,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:06,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54718 deadline: 1695968046818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:06,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:06,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968046836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:06,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:06,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968046920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:06,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:06,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54718 deadline: 1695968046931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:06,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:06,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968046937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:06,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:06,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968046937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:06,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:06,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968046941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:07,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:07,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968047124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:07,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:07,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54718 deadline: 1695968047139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:07,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:07,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968047145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:07,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:07,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968047144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:07,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:07,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968047148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:07,206 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/935432a161704c2b828ac0509a6523c2 2023-09-29T06:13:07,252 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/04ee45a0174b464fa2eaaa05e9b4dec7 is 50, key is test_row_0/B:col10/1695967986678/Put/seqid=0 2023-09-29T06:13:07,268 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2023-09-29T06:13:07,307 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/04ee45a0174b464fa2eaaa05e9b4dec7 2023-09-29T06:13:07,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/b60a1211e07149c8ab65dafad473c22a is 50, key is test_row_0/C:col10/1695967986678/Put/seqid=0 2023-09-29T06:13:07,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/b60a1211e07149c8ab65dafad473c22a 2023-09-29T06:13:07,420 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2023-09-29T06:13:07,421 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2023-09-29T06:13:07,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/935432a161704c2b828ac0509a6523c2 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/935432a161704c2b828ac0509a6523c2 2023-09-29T06:13:07,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:07,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968047433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:07,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:07,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54718 deadline: 1695968047446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:07,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:07,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968047450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:07,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:07,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968047453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:07,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:07,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968047455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:07,461 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2023-09-29T06:13:07,462 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2023-09-29T06:13:07,464 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/935432a161704c2b828ac0509a6523c2, entries=150, sequenceid=54, filesize=11.7 K 2023-09-29T06:13:07,464 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2023-09-29T06:13:07,464 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2023-09-29T06:13:07,467 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-09-29T06:13:07,467 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2023-09-29T06:13:07,468 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2023-09-29T06:13:07,468 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2023-09-29T06:13:07,469 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2023-09-29T06:13:07,469 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2023-09-29T06:13:07,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/04ee45a0174b464fa2eaaa05e9b4dec7 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/04ee45a0174b464fa2eaaa05e9b4dec7 2023-09-29T06:13:07,484 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/04ee45a0174b464fa2eaaa05e9b4dec7, entries=150, sequenceid=54, filesize=11.7 K 2023-09-29T06:13:07,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/b60a1211e07149c8ab65dafad473c22a as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/b60a1211e07149c8ab65dafad473c22a 2023-09-29T06:13:07,515 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/b60a1211e07149c8ab65dafad473c22a, entries=150, sequenceid=54, filesize=11.7 K 2023-09-29T06:13:07,527 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 1291e331f06575ebce0d38fd8ca0d9de in 846ms, sequenceid=54, compaction requested=true 2023-09-29T06:13:07,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:07,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1291e331f06575ebce0d38fd8ca0d9de:A, priority=-2147483648, current under compaction store size is 1 2023-09-29T06:13:07,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:07,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1291e331f06575ebce0d38fd8ca0d9de:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:13:07,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:07,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1291e331f06575ebce0d38fd8ca0d9de:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:13:07,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:13:07,535 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:13:07,536 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:13:07,552 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35952 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:13:07,554 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 1291e331f06575ebce0d38fd8ca0d9de/B is initiating minor compaction (all files) 2023-09-29T06:13:07,554 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1291e331f06575ebce0d38fd8ca0d9de/B in TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:07,555 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/5a1257a4902e435099f81cce4f0c7df9, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/2304f50233a444ccbb60f3b2de131c6f, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/04ee45a0174b464fa2eaaa05e9b4dec7] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp, totalSize=35.1 K 2023-09-29T06:13:07,556 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 5a1257a4902e435099f81cce4f0c7df9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1695967984203 2023-09-29T06:13:07,557 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 2304f50233a444ccbb60f3b2de131c6f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1695967984342 2023-09-29T06:13:07,558 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 04ee45a0174b464fa2eaaa05e9b4dec7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1695967985536 2023-09-29T06:13:07,565 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35952 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:13:07,566 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 1291e331f06575ebce0d38fd8ca0d9de/A is initiating minor compaction (all files) 2023-09-29T06:13:07,566 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1291e331f06575ebce0d38fd8ca0d9de/A in TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:07,566 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/f3f958725f1740dab741720d3eb189ad, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/5f6ff41a29b549c8b74e3642fe2fbe3f, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/935432a161704c2b828ac0509a6523c2] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp, totalSize=35.1 K 2023-09-29T06:13:07,569 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting f3f958725f1740dab741720d3eb189ad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1695967984203 2023-09-29T06:13:07,570 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 5f6ff41a29b549c8b74e3642fe2fbe3f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1695967984342 2023-09-29T06:13:07,571 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 935432a161704c2b828ac0509a6523c2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1695967985536 2023-09-29T06:13:07,607 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1291e331f06575ebce0d38fd8ca0d9de#B#compaction#9 average throughput is 0.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:07,608 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/967e0faf18684a8e9fd4cf8b4ec601fe is 50, key is test_row_0/B:col10/1695967986678/Put/seqid=0 2023-09-29T06:13:07,616 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1291e331f06575ebce0d38fd8ca0d9de#A#compaction#10 average throughput is 0.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:07,617 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/c6a8cd8f6dd6454688217395da8d0e35 is 50, key is test_row_0/A:col10/1695967986678/Put/seqid=0 2023-09-29T06:13:07,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 1291e331f06575ebce0d38fd8ca0d9de 2023-09-29T06:13:07,957 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 1291e331f06575ebce0d38fd8ca0d9de 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2023-09-29T06:13:07,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=A 2023-09-29T06:13:07,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:07,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=B 2023-09-29T06:13:07,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:07,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=C 2023-09-29T06:13:07,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:07,968 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/91ad6e696e5e45e7a21e5f24cc86fce9 is 50, key is test_row_0/A:col10/1695967986805/Put/seqid=0 2023-09-29T06:13:08,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:08,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968047988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:08,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:08,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968047991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:08,010 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/91ad6e696e5e45e7a21e5f24cc86fce9 2023-09-29T06:13:08,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:08,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54718 deadline: 1695968048000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:08,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:08,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:08,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968048007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:08,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968048007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:08,056 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/60c5c22f85a24de18be368dfff488055 is 50, key is test_row_0/B:col10/1695967986805/Put/seqid=0 2023-09-29T06:13:08,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:08,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968048119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:08,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:08,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968048119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:08,134 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/60c5c22f85a24de18be368dfff488055 2023-09-29T06:13:08,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:08,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968048125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:08,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:08,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54718 deadline: 1695968048132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:08,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:08,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968048135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:08,153 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/967e0faf18684a8e9fd4cf8b4ec601fe as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/967e0faf18684a8e9fd4cf8b4ec601fe 2023-09-29T06:13:08,169 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/c6a8cd8f6dd6454688217395da8d0e35 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/c6a8cd8f6dd6454688217395da8d0e35 2023-09-29T06:13:08,185 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/30a7353119a84c3bb52780ea67071fc0 is 50, key is test_row_0/C:col10/1695967986805/Put/seqid=0 2023-09-29T06:13:08,234 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 1291e331f06575ebce0d38fd8ca0d9de/B of 1291e331f06575ebce0d38fd8ca0d9de into 967e0faf18684a8e9fd4cf8b4ec601fe(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:08,234 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:08,234 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 1291e331f06575ebce0d38fd8ca0d9de/A of 1291e331f06575ebce0d38fd8ca0d9de into c6a8cd8f6dd6454688217395da8d0e35(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:08,234 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de., storeName=1291e331f06575ebce0d38fd8ca0d9de/B, priority=13, startTime=1695967987535; duration=0sec 2023-09-29T06:13:08,234 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:08,234 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de., storeName=1291e331f06575ebce0d38fd8ca0d9de/A, priority=13, startTime=1695967987530; duration=0sec 2023-09-29T06:13:08,236 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:13:08,236 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1291e331f06575ebce0d38fd8ca0d9de:A 2023-09-29T06:13:08,236 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:13:08,242 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:08,242 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1291e331f06575ebce0d38fd8ca0d9de:B 2023-09-29T06:13:08,255 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35952 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:13:08,255 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 1291e331f06575ebce0d38fd8ca0d9de/C is initiating minor compaction (all files) 2023-09-29T06:13:08,255 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1291e331f06575ebce0d38fd8ca0d9de/C in TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:08,256 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/ac24b6cd9df64fd9afa0d4bb1336af20, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/f6782c14c6df4083a91bb4f738ca167a, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/b60a1211e07149c8ab65dafad473c22a] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp, totalSize=35.1 K 2023-09-29T06:13:08,257 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting ac24b6cd9df64fd9afa0d4bb1336af20, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1695967984203 2023-09-29T06:13:08,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=12 2023-09-29T06:13:08,271 INFO [Thread-152 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 12 completed 2023-09-29T06:13:08,273 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:13:08,284 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting f6782c14c6df4083a91bb4f738ca167a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1695967984342 2023-09-29T06:13:08,286 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting b60a1211e07149c8ab65dafad473c22a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1695967985536 2023-09-29T06:13:08,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=14, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=14, table=TestAcidGuarantees 2023-09-29T06:13:08,289 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=14, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=14, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:13:08,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=14 2023-09-29T06:13:08,294 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=14, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=14, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:13:08,294 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:13:08,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:08,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968048327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:08,331 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1291e331f06575ebce0d38fd8ca0d9de#C#compaction#14 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:08,334 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/b00303588bd14d21860f5ead4cfaf899 is 50, key is test_row_0/C:col10/1695967986678/Put/seqid=0 2023-09-29T06:13:08,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:08,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968048326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:08,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:08,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968048338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:08,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:08,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54718 deadline: 1695968048340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:08,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:08,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968048342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:08,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=14 2023-09-29T06:13:08,407 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/b00303588bd14d21860f5ead4cfaf899 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/b00303588bd14d21860f5ead4cfaf899 2023-09-29T06:13:08,421 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 1291e331f06575ebce0d38fd8ca0d9de/C of 1291e331f06575ebce0d38fd8ca0d9de into b00303588bd14d21860f5ead4cfaf899(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:08,422 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:08,422 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de., storeName=1291e331f06575ebce0d38fd8ca0d9de/C, priority=13, startTime=1695967987535; duration=0sec 2023-09-29T06:13:08,422 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:08,422 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1291e331f06575ebce0d38fd8ca0d9de:C 2023-09-29T06:13:08,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=15 2023-09-29T06:13:08,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:08,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. as already flushing 2023-09-29T06:13:08,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:08,451 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] handler.RSProcedureHandler(53): pid=15 java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:08,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=15 java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:08,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=15 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:08,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=14 2023-09-29T06:13:08,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=15 2023-09-29T06:13:08,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:08,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. as already flushing 2023-09-29T06:13:08,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:08,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=15}] handler.RSProcedureHandler(53): pid=15 java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:08,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=15 java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:08,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=15 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:08,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:08,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968048636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:08,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:08,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968048639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:08,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:08,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968048644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:08,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:08,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968048649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:08,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:08,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54718 deadline: 1695968048649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:08,657 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/30a7353119a84c3bb52780ea67071fc0 2023-09-29T06:13:08,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/91ad6e696e5e45e7a21e5f24cc86fce9 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/91ad6e696e5e45e7a21e5f24cc86fce9 2023-09-29T06:13:08,677 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/91ad6e696e5e45e7a21e5f24cc86fce9, entries=150, sequenceid=74, filesize=11.7 K 2023-09-29T06:13:08,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/60c5c22f85a24de18be368dfff488055 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/60c5c22f85a24de18be368dfff488055 2023-09-29T06:13:08,703 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/60c5c22f85a24de18be368dfff488055, entries=150, sequenceid=74, filesize=11.7 K 2023-09-29T06:13:08,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/30a7353119a84c3bb52780ea67071fc0 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/30a7353119a84c3bb52780ea67071fc0 2023-09-29T06:13:08,724 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/30a7353119a84c3bb52780ea67071fc0, entries=150, sequenceid=74, filesize=11.7 K 2023-09-29T06:13:08,725 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 1291e331f06575ebce0d38fd8ca0d9de in 769ms, sequenceid=74, compaction requested=false 2023-09-29T06:13:08,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:08,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=15 2023-09-29T06:13:08,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:08,781 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegion(2819): Flushing 1291e331f06575ebce0d38fd8ca0d9de 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2023-09-29T06:13:08,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=A 2023-09-29T06:13:08,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:08,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=B 2023-09-29T06:13:08,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:08,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=C 2023-09-29T06:13:08,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:08,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/cda5389353e1479a923e096305a78960 is 50, key is test_row_0/A:col10/1695967987959/Put/seqid=0 2023-09-29T06:13:08,868 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/cda5389353e1479a923e096305a78960 2023-09-29T06:13:08,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/eff8c7cd4456421388b638a1ddd0bfe6 is 50, key is test_row_0/B:col10/1695967987959/Put/seqid=0 2023-09-29T06:13:08,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=14 2023-09-29T06:13:08,936 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/eff8c7cd4456421388b638a1ddd0bfe6 2023-09-29T06:13:08,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/ff4bf8adf82743c592d21169d667e216 is 50, key is test_row_0/C:col10/1695967987959/Put/seqid=0 2023-09-29T06:13:08,999 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/ff4bf8adf82743c592d21169d667e216 2023-09-29T06:13:09,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/cda5389353e1479a923e096305a78960 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/cda5389353e1479a923e096305a78960 2023-09-29T06:13:09,024 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/cda5389353e1479a923e096305a78960, entries=150, sequenceid=93, filesize=11.7 K 2023-09-29T06:13:09,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/eff8c7cd4456421388b638a1ddd0bfe6 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/eff8c7cd4456421388b638a1ddd0bfe6 2023-09-29T06:13:09,064 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/eff8c7cd4456421388b638a1ddd0bfe6, entries=150, sequenceid=93, filesize=11.7 K 2023-09-29T06:13:09,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/ff4bf8adf82743c592d21169d667e216 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/ff4bf8adf82743c592d21169d667e216 2023-09-29T06:13:09,089 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/ff4bf8adf82743c592d21169d667e216, entries=150, sequenceid=93, filesize=11.7 K 2023-09-29T06:13:09,091 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegion(3022): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=0 B/0 for 1291e331f06575ebce0d38fd8ca0d9de in 310ms, sequenceid=93, compaction requested=true 2023-09-29T06:13:09,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegion(2520): Flush status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:09,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:09,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=15 2023-09-29T06:13:09,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=15 2023-09-29T06:13:09,110 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=15, resume processing ppid=14 2023-09-29T06:13:09,110 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=15, ppid=14, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 813 msec 2023-09-29T06:13:09,113 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=14, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=14, table=TestAcidGuarantees in 836 msec 2023-09-29T06:13:09,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 1291e331f06575ebce0d38fd8ca0d9de 2023-09-29T06:13:09,166 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 1291e331f06575ebce0d38fd8ca0d9de 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2023-09-29T06:13:09,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=A 2023-09-29T06:13:09,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:09,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=B 2023-09-29T06:13:09,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:09,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=C 2023-09-29T06:13:09,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:09,177 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/d73ebafa2a5a4671b809f1d250d46c66 is 50, key is test_row_0/A:col10/1695967989162/Put/seqid=0 2023-09-29T06:13:09,258 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/d73ebafa2a5a4671b809f1d250d46c66 2023-09-29T06:13:09,277 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/703be8a3ae924bc4aae6f13f344f8eae is 50, key is test_row_0/B:col10/1695967989162/Put/seqid=0 2023-09-29T06:13:09,294 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/703be8a3ae924bc4aae6f13f344f8eae 2023-09-29T06:13:09,312 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/c999424ef138458c96260b379a430e94 is 50, key is test_row_0/C:col10/1695967989162/Put/seqid=0 2023-09-29T06:13:09,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:09,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968049307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:09,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:09,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968049310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:09,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:09,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54718 deadline: 1695968049314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:09,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:09,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968049328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:09,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:09,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968049328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:09,379 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/c999424ef138458c96260b379a430e94 2023-09-29T06:13:09,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/d73ebafa2a5a4671b809f1d250d46c66 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/d73ebafa2a5a4671b809f1d250d46c66 2023-09-29T06:13:09,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/d73ebafa2a5a4671b809f1d250d46c66, entries=300, sequenceid=104, filesize=18.6 K 2023-09-29T06:13:09,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=14 2023-09-29T06:13:09,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/703be8a3ae924bc4aae6f13f344f8eae as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/703be8a3ae924bc4aae6f13f344f8eae 2023-09-29T06:13:09,403 INFO [Thread-152 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 14 completed 2023-09-29T06:13:09,406 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:13:09,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=16, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=16, table=TestAcidGuarantees 2023-09-29T06:13:09,419 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=16, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=16, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:13:09,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-09-29T06:13:09,421 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=16, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=16, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:13:09,421 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:13:09,426 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/703be8a3ae924bc4aae6f13f344f8eae, entries=150, sequenceid=104, filesize=11.7 K 2023-09-29T06:13:09,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/c999424ef138458c96260b379a430e94 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/c999424ef138458c96260b379a430e94 2023-09-29T06:13:09,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:09,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:09,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54718 deadline: 1695968049422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:09,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968049421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:09,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:09,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968049429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:09,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:09,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968049435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:09,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:09,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968049435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:09,441 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/c999424ef138458c96260b379a430e94, entries=150, sequenceid=104, filesize=11.7 K 2023-09-29T06:13:09,444 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 1291e331f06575ebce0d38fd8ca0d9de in 278ms, sequenceid=104, compaction requested=true 2023-09-29T06:13:09,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:09,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1291e331f06575ebce0d38fd8ca0d9de:A, priority=-2147483648, current under compaction store size is 1 2023-09-29T06:13:09,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:09,444 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-29T06:13:09,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1291e331f06575ebce0d38fd8ca0d9de:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:13:09,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:09,444 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-29T06:13:09,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1291e331f06575ebce0d38fd8ca0d9de:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:13:09,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:13:09,447 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55059 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-29T06:13:09,447 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48039 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-29T06:13:09,447 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 1291e331f06575ebce0d38fd8ca0d9de/A is initiating minor compaction (all files) 2023-09-29T06:13:09,447 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 1291e331f06575ebce0d38fd8ca0d9de/B is initiating minor compaction (all files) 2023-09-29T06:13:09,447 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1291e331f06575ebce0d38fd8ca0d9de/A in TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:09,447 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1291e331f06575ebce0d38fd8ca0d9de/B in TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:09,447 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/967e0faf18684a8e9fd4cf8b4ec601fe, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/60c5c22f85a24de18be368dfff488055, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/eff8c7cd4456421388b638a1ddd0bfe6, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/703be8a3ae924bc4aae6f13f344f8eae] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp, totalSize=46.9 K 2023-09-29T06:13:09,447 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/c6a8cd8f6dd6454688217395da8d0e35, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/91ad6e696e5e45e7a21e5f24cc86fce9, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/cda5389353e1479a923e096305a78960, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/d73ebafa2a5a4671b809f1d250d46c66] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp, totalSize=53.8 K 2023-09-29T06:13:09,448 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 967e0faf18684a8e9fd4cf8b4ec601fe, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1695967985536 2023-09-29T06:13:09,448 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting c6a8cd8f6dd6454688217395da8d0e35, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1695967985536 2023-09-29T06:13:09,448 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 60c5c22f85a24de18be368dfff488055, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1695967986794 2023-09-29T06:13:09,449 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 91ad6e696e5e45e7a21e5f24cc86fce9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1695967986794 2023-09-29T06:13:09,449 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting eff8c7cd4456421388b638a1ddd0bfe6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1695967987959 2023-09-29T06:13:09,450 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting cda5389353e1479a923e096305a78960, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1695967987959 2023-09-29T06:13:09,450 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 703be8a3ae924bc4aae6f13f344f8eae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1695967989160 2023-09-29T06:13:09,450 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting d73ebafa2a5a4671b809f1d250d46c66, keycount=300, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1695967989153 2023-09-29T06:13:09,501 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1291e331f06575ebce0d38fd8ca0d9de#A#compaction#21 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:09,502 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/d70a7386a8d742848ee85a3b29018a78 is 50, key is test_row_0/A:col10/1695967989162/Put/seqid=0 2023-09-29T06:13:09,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-09-29T06:13:09,532 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1291e331f06575ebce0d38fd8ca0d9de#B#compaction#22 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:09,533 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/69c095194f404f9f976b50c0d70af93a is 50, key is test_row_0/B:col10/1695967989162/Put/seqid=0 2023-09-29T06:13:09,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=17 2023-09-29T06:13:09,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:09,579 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegion(2819): Flushing 1291e331f06575ebce0d38fd8ca0d9de 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2023-09-29T06:13:09,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=A 2023-09-29T06:13:09,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:09,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=B 2023-09-29T06:13:09,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:09,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=C 2023-09-29T06:13:09,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:09,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/ba0ab8d9e3ba4a3997ce33c74fd0f0f9 is 50, key is test_row_0/A:col10/1695967989244/Put/seqid=0 2023-09-29T06:13:09,618 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/d70a7386a8d742848ee85a3b29018a78 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/d70a7386a8d742848ee85a3b29018a78 2023-09-29T06:13:09,640 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 1291e331f06575ebce0d38fd8ca0d9de/A of 1291e331f06575ebce0d38fd8ca0d9de into d70a7386a8d742848ee85a3b29018a78(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:09,640 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:09,640 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de., storeName=1291e331f06575ebce0d38fd8ca0d9de/A, priority=12, startTime=1695967989444; duration=0sec 2023-09-29T06:13:09,640 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:13:09,640 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1291e331f06575ebce0d38fd8ca0d9de:A 2023-09-29T06:13:09,640 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-29T06:13:09,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 1291e331f06575ebce0d38fd8ca0d9de 2023-09-29T06:13:09,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. as already flushing 2023-09-29T06:13:09,644 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48039 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-29T06:13:09,644 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 1291e331f06575ebce0d38fd8ca0d9de/C is initiating minor compaction (all files) 2023-09-29T06:13:09,645 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1291e331f06575ebce0d38fd8ca0d9de/C in TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:09,645 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/b00303588bd14d21860f5ead4cfaf899, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/30a7353119a84c3bb52780ea67071fc0, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/ff4bf8adf82743c592d21169d667e216, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/c999424ef138458c96260b379a430e94] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp, totalSize=46.9 K 2023-09-29T06:13:09,654 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting b00303588bd14d21860f5ead4cfaf899, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1695967985536 2023-09-29T06:13:09,655 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 30a7353119a84c3bb52780ea67071fc0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1695967986794 2023-09-29T06:13:09,657 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting ff4bf8adf82743c592d21169d667e216, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1695967987959 2023-09-29T06:13:09,658 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting c999424ef138458c96260b379a430e94, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1695967989160 2023-09-29T06:13:09,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:09,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968049661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:09,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:09,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968049659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:09,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:09,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968049668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:09,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:09,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54718 deadline: 1695968049669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:09,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:09,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968049667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:09,680 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1291e331f06575ebce0d38fd8ca0d9de#C#compaction#24 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:09,680 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/ce3390af828e4e4e815984f24c1a20d8 is 50, key is test_row_0/C:col10/1695967989162/Put/seqid=0 2023-09-29T06:13:09,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-09-29T06:13:09,751 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/ce3390af828e4e4e815984f24c1a20d8 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/ce3390af828e4e4e815984f24c1a20d8 2023-09-29T06:13:09,771 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 1291e331f06575ebce0d38fd8ca0d9de/C of 1291e331f06575ebce0d38fd8ca0d9de into ce3390af828e4e4e815984f24c1a20d8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:09,771 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:09,771 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de., storeName=1291e331f06575ebce0d38fd8ca0d9de/C, priority=12, startTime=1695967989444; duration=0sec 2023-09-29T06:13:09,771 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:09,771 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1291e331f06575ebce0d38fd8ca0d9de:C 2023-09-29T06:13:09,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:09,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968049771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:09,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:09,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968049776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:09,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:09,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968049776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:09,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:09,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54718 deadline: 1695968049777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:09,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:09,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968049779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:09,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:09,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968049975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:09,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:09,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968049987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:09,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:09,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968049987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:09,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:09,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54718 deadline: 1695968049988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:09,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:09,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968049989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:10,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-09-29T06:13:10,057 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/69c095194f404f9f976b50c0d70af93a as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/69c095194f404f9f976b50c0d70af93a 2023-09-29T06:13:10,059 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/ba0ab8d9e3ba4a3997ce33c74fd0f0f9 2023-09-29T06:13:10,078 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 1291e331f06575ebce0d38fd8ca0d9de/B of 1291e331f06575ebce0d38fd8ca0d9de into 69c095194f404f9f976b50c0d70af93a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:10,078 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:10,078 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de., storeName=1291e331f06575ebce0d38fd8ca0d9de/B, priority=12, startTime=1695967989444; duration=0sec 2023-09-29T06:13:10,078 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:10,079 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1291e331f06575ebce0d38fd8ca0d9de:B 2023-09-29T06:13:10,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/578f75685c56486786749cf5b272a3aa is 50, key is test_row_0/B:col10/1695967989244/Put/seqid=0 2023-09-29T06:13:10,119 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/578f75685c56486786749cf5b272a3aa 2023-09-29T06:13:10,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/fc0c22d481bc4b75a65451a5739385b8 is 50, key is test_row_0/C:col10/1695967989244/Put/seqid=0 2023-09-29T06:13:10,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:10,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968050279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:10,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:10,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968050293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:10,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:10,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:10,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968050293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:10,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968050293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:10,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:10,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54718 deadline: 1695968050293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:10,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-09-29T06:13:10,570 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/fc0c22d481bc4b75a65451a5739385b8 2023-09-29T06:13:10,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/ba0ab8d9e3ba4a3997ce33c74fd0f0f9 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/ba0ab8d9e3ba4a3997ce33c74fd0f0f9 2023-09-29T06:13:10,590 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/ba0ab8d9e3ba4a3997ce33c74fd0f0f9, entries=150, sequenceid=130, filesize=11.8 K 2023-09-29T06:13:10,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/578f75685c56486786749cf5b272a3aa as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/578f75685c56486786749cf5b272a3aa 2023-09-29T06:13:10,605 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/578f75685c56486786749cf5b272a3aa, entries=150, sequenceid=130, filesize=11.8 K 2023-09-29T06:13:10,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/fc0c22d481bc4b75a65451a5739385b8 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/fc0c22d481bc4b75a65451a5739385b8 2023-09-29T06:13:10,616 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/fc0c22d481bc4b75a65451a5739385b8, entries=150, sequenceid=130, filesize=11.8 K 2023-09-29T06:13:10,623 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegion(3022): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 1291e331f06575ebce0d38fd8ca0d9de in 1045ms, sequenceid=130, compaction requested=false 2023-09-29T06:13:10,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegion(2520): Flush status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:10,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:10,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=17 2023-09-29T06:13:10,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=17 2023-09-29T06:13:10,637 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=17, resume processing ppid=16 2023-09-29T06:13:10,637 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=17, ppid=16, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2120 sec 2023-09-29T06:13:10,640 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=16, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=16, table=TestAcidGuarantees in 1.2320 sec 2023-09-29T06:13:10,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 1291e331f06575ebce0d38fd8ca0d9de 2023-09-29T06:13:10,788 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 1291e331f06575ebce0d38fd8ca0d9de 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2023-09-29T06:13:10,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=A 2023-09-29T06:13:10,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:10,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=B 2023-09-29T06:13:10,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:10,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=C 2023-09-29T06:13:10,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:10,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/b081b9464fe84a27ad21fdbd94ac614d is 50, key is test_row_0/A:col10/1695967989663/Put/seqid=0 2023-09-29T06:13:10,851 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/b081b9464fe84a27ad21fdbd94ac614d 2023-09-29T06:13:10,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:10,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968050845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:10,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:10,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968050845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:10,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:10,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968050852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:10,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:10,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54718 deadline: 1695968050852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:10,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:10,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968050859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:10,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/4bacee4e8d0f401d8445b0d2f9459187 is 50, key is test_row_0/B:col10/1695967989663/Put/seqid=0 2023-09-29T06:13:10,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:10,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968050954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:10,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:10,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968050954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:10,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:10,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968050958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:10,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:10,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54718 deadline: 1695968050960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:10,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:10,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968050962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:11,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:11,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968051157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:11,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:11,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968051158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:11,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:11,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968051162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:11,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:11,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54718 deadline: 1695968051165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:11,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:11,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968051168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:11,290 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/4bacee4e8d0f401d8445b0d2f9459187 2023-09-29T06:13:11,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:11,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968051465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:11,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:11,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968051465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:11,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:11,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968051466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:11,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:11,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54718 deadline: 1695968051471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:11,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:11,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968051475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:11,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/2252230c61bf44749d4d0b3def1ef74e is 50, key is test_row_0/C:col10/1695967989663/Put/seqid=0 2023-09-29T06:13:11,506 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/2252230c61bf44749d4d0b3def1ef74e 2023-09-29T06:13:11,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/b081b9464fe84a27ad21fdbd94ac614d as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/b081b9464fe84a27ad21fdbd94ac614d 2023-09-29T06:13:11,523 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/b081b9464fe84a27ad21fdbd94ac614d, entries=150, sequenceid=145, filesize=11.8 K 2023-09-29T06:13:11,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-09-29T06:13:11,527 INFO [Thread-152 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 16 completed 2023-09-29T06:13:11,530 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:13:11,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=18, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=18, table=TestAcidGuarantees 2023-09-29T06:13:11,534 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=18, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=18, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:13:11,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/4bacee4e8d0f401d8445b0d2f9459187 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/4bacee4e8d0f401d8445b0d2f9459187 2023-09-29T06:13:11,535 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=18, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=18, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:13:11,535 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:13:11,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=18 2023-09-29T06:13:11,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/4bacee4e8d0f401d8445b0d2f9459187, entries=150, sequenceid=145, filesize=11.8 K 2023-09-29T06:13:11,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/2252230c61bf44749d4d0b3def1ef74e as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/2252230c61bf44749d4d0b3def1ef74e 2023-09-29T06:13:11,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/2252230c61bf44749d4d0b3def1ef74e, entries=150, sequenceid=145, filesize=11.8 K 2023-09-29T06:13:11,593 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 1291e331f06575ebce0d38fd8ca0d9de in 805ms, sequenceid=145, compaction requested=true 2023-09-29T06:13:11,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:11,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1291e331f06575ebce0d38fd8ca0d9de:A, priority=-2147483648, current under compaction store size is 1 2023-09-29T06:13:11,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:11,593 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:13:11,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1291e331f06575ebce0d38fd8ca0d9de:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:13:11,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:11,594 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:13:11,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1291e331f06575ebce0d38fd8ca0d9de:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:13:11,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:13:11,596 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36392 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:13:11,596 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36392 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:13:11,596 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 1291e331f06575ebce0d38fd8ca0d9de/B is initiating minor compaction (all files) 2023-09-29T06:13:11,596 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 1291e331f06575ebce0d38fd8ca0d9de/A is initiating minor compaction (all files) 2023-09-29T06:13:11,596 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1291e331f06575ebce0d38fd8ca0d9de/B in TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:11,596 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1291e331f06575ebce0d38fd8ca0d9de/A in TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:11,596 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/69c095194f404f9f976b50c0d70af93a, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/578f75685c56486786749cf5b272a3aa, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/4bacee4e8d0f401d8445b0d2f9459187] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp, totalSize=35.5 K 2023-09-29T06:13:11,596 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/d70a7386a8d742848ee85a3b29018a78, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/ba0ab8d9e3ba4a3997ce33c74fd0f0f9, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/b081b9464fe84a27ad21fdbd94ac614d] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp, totalSize=35.5 K 2023-09-29T06:13:11,597 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 69c095194f404f9f976b50c0d70af93a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1695967989160 2023-09-29T06:13:11,597 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting d70a7386a8d742848ee85a3b29018a78, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1695967989160 2023-09-29T06:13:11,598 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 578f75685c56486786749cf5b272a3aa, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1695967989244 2023-09-29T06:13:11,598 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting ba0ab8d9e3ba4a3997ce33c74fd0f0f9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1695967989244 2023-09-29T06:13:11,598 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 4bacee4e8d0f401d8445b0d2f9459187, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1695967989663 2023-09-29T06:13:11,599 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting b081b9464fe84a27ad21fdbd94ac614d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1695967989663 2023-09-29T06:13:11,612 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1291e331f06575ebce0d38fd8ca0d9de#B#compaction#30 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:11,612 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/db46cd714ab44c03a88925e24c5929a1 is 50, key is test_row_0/B:col10/1695967989663/Put/seqid=0 2023-09-29T06:13:11,636 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1291e331f06575ebce0d38fd8ca0d9de#A#compaction#31 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:11,637 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/d03fde6465284b888486d9e5f8610c74 is 50, key is test_row_0/A:col10/1695967989663/Put/seqid=0 2023-09-29T06:13:11,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=18 2023-09-29T06:13:11,676 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/d03fde6465284b888486d9e5f8610c74 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/d03fde6465284b888486d9e5f8610c74 2023-09-29T06:13:11,693 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 1291e331f06575ebce0d38fd8ca0d9de/A of 1291e331f06575ebce0d38fd8ca0d9de into d03fde6465284b888486d9e5f8610c74(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:11,693 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:11,693 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de., storeName=1291e331f06575ebce0d38fd8ca0d9de/A, priority=13, startTime=1695967991593; duration=0sec 2023-09-29T06:13:11,693 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:13:11,693 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1291e331f06575ebce0d38fd8ca0d9de:A 2023-09-29T06:13:11,693 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:13:11,695 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36392 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:13:11,695 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 1291e331f06575ebce0d38fd8ca0d9de/C is initiating minor compaction (all files) 2023-09-29T06:13:11,695 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1291e331f06575ebce0d38fd8ca0d9de/C in TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:11,695 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/ce3390af828e4e4e815984f24c1a20d8, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/fc0c22d481bc4b75a65451a5739385b8, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/2252230c61bf44749d4d0b3def1ef74e] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp, totalSize=35.5 K 2023-09-29T06:13:11,696 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting ce3390af828e4e4e815984f24c1a20d8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1695967989160 2023-09-29T06:13:11,696 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting fc0c22d481bc4b75a65451a5739385b8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1695967989244 2023-09-29T06:13:11,703 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 2252230c61bf44749d4d0b3def1ef74e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1695967989663 2023-09-29T06:13:11,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=19 2023-09-29T06:13:11,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:11,704 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegion(2819): Flushing 1291e331f06575ebce0d38fd8ca0d9de 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2023-09-29T06:13:11,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=A 2023-09-29T06:13:11,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:11,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=B 2023-09-29T06:13:11,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:11,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=C 2023-09-29T06:13:11,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:11,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/3003e4dbd3ba4ba7b2da9e8a40cebeae is 50, key is test_row_0/A:col10/1695967990840/Put/seqid=0 2023-09-29T06:13:11,732 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1291e331f06575ebce0d38fd8ca0d9de#C#compaction#33 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:11,734 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/e33f6a4efb36431198f0ef5ab7ace513 is 50, key is test_row_0/C:col10/1695967989663/Put/seqid=0 2023-09-29T06:13:11,770 WARN [DataStreamer for file /user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/e33f6a4efb36431198f0ef5ab7ace513 {}] hdfs.DataStreamer(982): Caught exception java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:1.8.0_362] at java.lang.Thread.join(Thread.java:1257) ~[?:1.8.0_362] at java.lang.Thread.join(Thread.java:1331) ~[?:1.8.0_362] at org.apache.hadoop.hdfs.DataStreamer.closeResponder(DataStreamer.java:980) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DataStreamer.endBlock(DataStreamer.java:630) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:807) ~[hadoop-hdfs-client-2.10.2.jar:?] 2023-09-29T06:13:11,771 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/3003e4dbd3ba4ba7b2da9e8a40cebeae 2023-09-29T06:13:11,784 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/e33f6a4efb36431198f0ef5ab7ace513 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/e33f6a4efb36431198f0ef5ab7ace513 2023-09-29T06:13:11,794 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 1291e331f06575ebce0d38fd8ca0d9de/C of 1291e331f06575ebce0d38fd8ca0d9de into e33f6a4efb36431198f0ef5ab7ace513(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:11,794 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:11,794 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de., storeName=1291e331f06575ebce0d38fd8ca0d9de/C, priority=13, startTime=1695967991594; duration=0sec 2023-09-29T06:13:11,795 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:11,796 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1291e331f06575ebce0d38fd8ca0d9de:C 2023-09-29T06:13:11,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/763ac22df391492e9777e19ab4050d7e is 50, key is test_row_0/B:col10/1695967990840/Put/seqid=0 2023-09-29T06:13:11,828 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/763ac22df391492e9777e19ab4050d7e 2023-09-29T06:13:11,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=18 2023-09-29T06:13:11,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/e95a249b75bb41d7a6d3160397fc845a is 50, key is test_row_0/C:col10/1695967990840/Put/seqid=0 2023-09-29T06:13:11,859 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/e95a249b75bb41d7a6d3160397fc845a 2023-09-29T06:13:11,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/3003e4dbd3ba4ba7b2da9e8a40cebeae as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/3003e4dbd3ba4ba7b2da9e8a40cebeae 2023-09-29T06:13:11,875 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/3003e4dbd3ba4ba7b2da9e8a40cebeae, entries=150, sequenceid=171, filesize=11.8 K 2023-09-29T06:13:11,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/763ac22df391492e9777e19ab4050d7e as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/763ac22df391492e9777e19ab4050d7e 2023-09-29T06:13:11,892 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/763ac22df391492e9777e19ab4050d7e, entries=150, sequenceid=171, filesize=11.8 K 2023-09-29T06:13:11,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/e95a249b75bb41d7a6d3160397fc845a as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/e95a249b75bb41d7a6d3160397fc845a 2023-09-29T06:13:11,912 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/e95a249b75bb41d7a6d3160397fc845a, entries=150, sequenceid=171, filesize=11.8 K 2023-09-29T06:13:11,914 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegion(3022): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for 1291e331f06575ebce0d38fd8ca0d9de in 209ms, sequenceid=171, compaction requested=false 2023-09-29T06:13:11,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegion(2520): Flush status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:11,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:11,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=19 2023-09-29T06:13:11,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=19 2023-09-29T06:13:11,931 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=19, resume processing ppid=18 2023-09-29T06:13:11,931 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=19, ppid=18, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 389 msec 2023-09-29T06:13:11,934 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=18, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=18, table=TestAcidGuarantees in 402 msec 2023-09-29T06:13:11,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 1291e331f06575ebce0d38fd8ca0d9de 2023-09-29T06:13:11,988 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 1291e331f06575ebce0d38fd8ca0d9de 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2023-09-29T06:13:11,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=A 2023-09-29T06:13:11,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:11,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=B 2023-09-29T06:13:11,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:11,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=C 2023-09-29T06:13:11,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:11,996 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/6b67e203054b46209699512d71ab24a2 is 50, key is test_row_0/A:col10/1695967991986/Put/seqid=0 2023-09-29T06:13:12,055 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/db46cd714ab44c03a88925e24c5929a1 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/db46cd714ab44c03a88925e24c5929a1 2023-09-29T06:13:12,072 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 1291e331f06575ebce0d38fd8ca0d9de/B of 1291e331f06575ebce0d38fd8ca0d9de into db46cd714ab44c03a88925e24c5929a1(size=12.2 K), total size for store is 24.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:12,072 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:12,072 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de., storeName=1291e331f06575ebce0d38fd8ca0d9de/B, priority=13, startTime=1695967991593; duration=0sec 2023-09-29T06:13:12,072 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:12,072 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1291e331f06575ebce0d38fd8ca0d9de:B 2023-09-29T06:13:12,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:12,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968052073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:12,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:12,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968052073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:12,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:12,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968052073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:12,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:12,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968052076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:12,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:12,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54718 deadline: 1695968052076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:12,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=18 2023-09-29T06:13:12,142 INFO [Thread-152 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 18 completed 2023-09-29T06:13:12,144 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:13:12,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=20, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=20, table=TestAcidGuarantees 2023-09-29T06:13:12,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=20 2023-09-29T06:13:12,148 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=20, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=20, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:13:12,149 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=20, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=20, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:13:12,149 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:13:12,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:12,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968052181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:12,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:12,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968052182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:12,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:12,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968052186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:12,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:12,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968052191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:12,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:12,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54718 deadline: 1695968052192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:12,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=20 2023-09-29T06:13:12,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=21 2023-09-29T06:13:12,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:12,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. as already flushing 2023-09-29T06:13:12,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:12,305 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] handler.RSProcedureHandler(53): pid=21 java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:12,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=21 java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:12,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=21 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:12,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:12,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968052385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:12,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:12,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968052385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:12,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:12,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968052392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:12,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:12,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968052395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:12,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:12,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54718 deadline: 1695968052397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:12,427 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/6b67e203054b46209699512d71ab24a2 2023-09-29T06:13:12,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/a05d7d2fe03045b8a6aefc8cdb56ab65 is 50, key is test_row_0/B:col10/1695967991986/Put/seqid=0 2023-09-29T06:13:12,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=20 2023-09-29T06:13:12,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=21 2023-09-29T06:13:12,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:12,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. as already flushing 2023-09-29T06:13:12,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:12,463 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] handler.RSProcedureHandler(53): pid=21 java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:12,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=21 java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:12,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=21 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:12,465 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/a05d7d2fe03045b8a6aefc8cdb56ab65 2023-09-29T06:13:12,481 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/ba5a1e2932cb44a193cd52e4f667791d is 50, key is test_row_0/C:col10/1695967991986/Put/seqid=0 2023-09-29T06:13:12,512 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/ba5a1e2932cb44a193cd52e4f667791d 2023-09-29T06:13:12,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/6b67e203054b46209699512d71ab24a2 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/6b67e203054b46209699512d71ab24a2 2023-09-29T06:13:12,533 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/6b67e203054b46209699512d71ab24a2, entries=200, sequenceid=183, filesize=14.2 K 2023-09-29T06:13:12,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/a05d7d2fe03045b8a6aefc8cdb56ab65 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/a05d7d2fe03045b8a6aefc8cdb56ab65 2023-09-29T06:13:12,544 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/a05d7d2fe03045b8a6aefc8cdb56ab65, entries=150, sequenceid=183, filesize=11.8 K 2023-09-29T06:13:12,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/ba5a1e2932cb44a193cd52e4f667791d as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/ba5a1e2932cb44a193cd52e4f667791d 2023-09-29T06:13:12,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/ba5a1e2932cb44a193cd52e4f667791d, entries=150, sequenceid=183, filesize=11.8 K 2023-09-29T06:13:12,561 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 1291e331f06575ebce0d38fd8ca0d9de in 572ms, sequenceid=183, compaction requested=true 2023-09-29T06:13:12,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:12,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1291e331f06575ebce0d38fd8ca0d9de:A, priority=-2147483648, current under compaction store size is 1 2023-09-29T06:13:12,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:12,561 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:13:12,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1291e331f06575ebce0d38fd8ca0d9de:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:13:12,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:13:12,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1291e331f06575ebce0d38fd8ca0d9de:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:13:12,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:13:12,561 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:13:12,565 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36744 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:13:12,565 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39134 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:13:12,565 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 1291e331f06575ebce0d38fd8ca0d9de/B is initiating minor compaction (all files) 2023-09-29T06:13:12,565 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 1291e331f06575ebce0d38fd8ca0d9de/A is initiating minor compaction (all files) 2023-09-29T06:13:12,566 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1291e331f06575ebce0d38fd8ca0d9de/B in TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:12,566 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1291e331f06575ebce0d38fd8ca0d9de/A in TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:12,566 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/db46cd714ab44c03a88925e24c5929a1, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/763ac22df391492e9777e19ab4050d7e, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/a05d7d2fe03045b8a6aefc8cdb56ab65] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp, totalSize=35.9 K 2023-09-29T06:13:12,566 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/d03fde6465284b888486d9e5f8610c74, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/3003e4dbd3ba4ba7b2da9e8a40cebeae, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/6b67e203054b46209699512d71ab24a2] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp, totalSize=38.2 K 2023-09-29T06:13:12,568 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting db46cd714ab44c03a88925e24c5929a1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1695967989663 2023-09-29T06:13:12,568 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting d03fde6465284b888486d9e5f8610c74, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1695967989663 2023-09-29T06:13:12,569 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 763ac22df391492e9777e19ab4050d7e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1695967990840 2023-09-29T06:13:12,569 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 3003e4dbd3ba4ba7b2da9e8a40cebeae, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1695967990840 2023-09-29T06:13:12,570 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 6b67e203054b46209699512d71ab24a2, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1695967991979 2023-09-29T06:13:12,570 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting a05d7d2fe03045b8a6aefc8cdb56ab65, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1695967991979 2023-09-29T06:13:12,583 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1291e331f06575ebce0d38fd8ca0d9de#A#compaction#39 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:12,584 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/030789cb8ca74e25b544c375b0dc3038 is 50, key is test_row_0/A:col10/1695967991986/Put/seqid=0 2023-09-29T06:13:12,584 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1291e331f06575ebce0d38fd8ca0d9de#B#compaction#40 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:12,585 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/2d8a1971ac6848489cf4366da08a4bc8 is 50, key is test_row_0/B:col10/1695967991986/Put/seqid=0 2023-09-29T06:13:12,621 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/2d8a1971ac6848489cf4366da08a4bc8 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/2d8a1971ac6848489cf4366da08a4bc8 2023-09-29T06:13:12,629 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/030789cb8ca74e25b544c375b0dc3038 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/030789cb8ca74e25b544c375b0dc3038 2023-09-29T06:13:12,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=21 2023-09-29T06:13:12,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:12,631 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(2819): Flushing 1291e331f06575ebce0d38fd8ca0d9de 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2023-09-29T06:13:12,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=A 2023-09-29T06:13:12,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:12,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=B 2023-09-29T06:13:12,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:12,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=C 2023-09-29T06:13:12,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:12,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/e1d644c0965f4fa9a54ded5b8da40c0d is 50, key is test_row_0/A:col10/1695967992006/Put/seqid=0 2023-09-29T06:13:12,642 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 1291e331f06575ebce0d38fd8ca0d9de/B of 1291e331f06575ebce0d38fd8ca0d9de into 2d8a1971ac6848489cf4366da08a4bc8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:12,642 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:12,642 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de., storeName=1291e331f06575ebce0d38fd8ca0d9de/B, priority=13, startTime=1695967992561; duration=0sec 2023-09-29T06:13:12,642 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:13:12,642 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1291e331f06575ebce0d38fd8ca0d9de:B 2023-09-29T06:13:12,642 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:13:12,644 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 1291e331f06575ebce0d38fd8ca0d9de/A of 1291e331f06575ebce0d38fd8ca0d9de into 030789cb8ca74e25b544c375b0dc3038(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:12,644 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:12,644 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de., storeName=1291e331f06575ebce0d38fd8ca0d9de/A, priority=13, startTime=1695967992561; duration=0sec 2023-09-29T06:13:12,644 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36744 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:13:12,644 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 1291e331f06575ebce0d38fd8ca0d9de/C is initiating minor compaction (all files) 2023-09-29T06:13:12,644 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:12,644 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1291e331f06575ebce0d38fd8ca0d9de:A 2023-09-29T06:13:12,644 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1291e331f06575ebce0d38fd8ca0d9de/C in TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:12,645 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/e33f6a4efb36431198f0ef5ab7ace513, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/e95a249b75bb41d7a6d3160397fc845a, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/ba5a1e2932cb44a193cd52e4f667791d] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp, totalSize=35.9 K 2023-09-29T06:13:12,645 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting e33f6a4efb36431198f0ef5ab7ace513, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1695967989663 2023-09-29T06:13:12,646 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting e95a249b75bb41d7a6d3160397fc845a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1695967990840 2023-09-29T06:13:12,647 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting ba5a1e2932cb44a193cd52e4f667791d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1695967991979 2023-09-29T06:13:12,670 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1291e331f06575ebce0d38fd8ca0d9de#C#compaction#42 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:12,671 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/e0105ac4698844b4a818e17907d613b1 is 50, key is test_row_0/C:col10/1695967991986/Put/seqid=0 2023-09-29T06:13:12,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 1291e331f06575ebce0d38fd8ca0d9de 2023-09-29T06:13:12,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. as already flushing 2023-09-29T06:13:12,700 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/e0105ac4698844b4a818e17907d613b1 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/e0105ac4698844b4a818e17907d613b1 2023-09-29T06:13:12,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:12,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968052702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:12,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:12,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968052707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:12,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:12,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54718 deadline: 1695968052703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:12,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:12,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968052707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:12,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:12,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968052708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:12,713 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 1291e331f06575ebce0d38fd8ca0d9de/C of 1291e331f06575ebce0d38fd8ca0d9de into e0105ac4698844b4a818e17907d613b1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:12,713 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:12,713 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de., storeName=1291e331f06575ebce0d38fd8ca0d9de/C, priority=13, startTime=1695967992561; duration=0sec 2023-09-29T06:13:12,713 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:12,713 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1291e331f06575ebce0d38fd8ca0d9de:C 2023-09-29T06:13:12,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=20 2023-09-29T06:13:12,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:12,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968052809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:12,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:12,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968052813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:12,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:12,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968052814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:13,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:13,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968053014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:13,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:13,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968053017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:13,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:13,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968053018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:13,075 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/e1d644c0965f4fa9a54ded5b8da40c0d 2023-09-29T06:13:13,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/3ab427fd69d5438fb058501d343cdbfb is 50, key is test_row_0/B:col10/1695967992006/Put/seqid=0 2023-09-29T06:13:13,129 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/3ab427fd69d5438fb058501d343cdbfb 2023-09-29T06:13:13,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/3078da1ea1684bd48536ca94fc6500f8 is 50, key is test_row_0/C:col10/1695967992006/Put/seqid=0 2023-09-29T06:13:13,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:13,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968053213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:13,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:13,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54718 deadline: 1695968053215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:13,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=20 2023-09-29T06:13:13,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:13,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968053317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:13,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:13,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968053322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:13,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:13,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968053322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:13,568 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/3078da1ea1684bd48536ca94fc6500f8 2023-09-29T06:13:13,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/e1d644c0965f4fa9a54ded5b8da40c0d as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/e1d644c0965f4fa9a54ded5b8da40c0d 2023-09-29T06:13:13,591 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/e1d644c0965f4fa9a54ded5b8da40c0d, entries=150, sequenceid=210, filesize=11.8 K 2023-09-29T06:13:13,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/3ab427fd69d5438fb058501d343cdbfb as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/3ab427fd69d5438fb058501d343cdbfb 2023-09-29T06:13:13,600 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/3ab427fd69d5438fb058501d343cdbfb, entries=150, sequenceid=210, filesize=11.8 K 2023-09-29T06:13:13,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/3078da1ea1684bd48536ca94fc6500f8 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/3078da1ea1684bd48536ca94fc6500f8 2023-09-29T06:13:13,616 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/3078da1ea1684bd48536ca94fc6500f8, entries=150, sequenceid=210, filesize=11.8 K 2023-09-29T06:13:13,618 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(3022): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 1291e331f06575ebce0d38fd8ca0d9de in 986ms, sequenceid=210, compaction requested=false 2023-09-29T06:13:13,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(2520): Flush status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:13,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:13,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=21 2023-09-29T06:13:13,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=21 2023-09-29T06:13:13,624 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=21, resume processing ppid=20 2023-09-29T06:13:13,624 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=21, ppid=20, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4720 sec 2023-09-29T06:13:13,634 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=20, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=20, table=TestAcidGuarantees in 1.4810 sec 2023-09-29T06:13:13,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 1291e331f06575ebce0d38fd8ca0d9de 2023-09-29T06:13:13,821 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 1291e331f06575ebce0d38fd8ca0d9de 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2023-09-29T06:13:13,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=A 2023-09-29T06:13:13,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:13,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=B 2023-09-29T06:13:13,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:13,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=C 2023-09-29T06:13:13,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:13,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/a38030f69f774fdabc0c907151a01784 is 50, key is test_row_0/A:col10/1695967993820/Put/seqid=0 2023-09-29T06:13:13,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:13,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968053895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:13,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:13,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968053898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:13,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:13,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968053902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:14,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:14,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968054000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:14,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:14,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968054002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:14,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:14,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968054007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:14,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:14,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968054205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:14,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:14,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968054205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:14,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:14,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968054210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:14,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:14,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54718 deadline: 1695968054225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:14,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:14,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968054240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:14,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=20 2023-09-29T06:13:14,262 INFO [Thread-152 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 20 completed 2023-09-29T06:13:14,263 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/a38030f69f774fdabc0c907151a01784 2023-09-29T06:13:14,266 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:13:14,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=22, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=22, table=TestAcidGuarantees 2023-09-29T06:13:14,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-09-29T06:13:14,273 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=22, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=22, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:13:14,276 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=22, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=22, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:13:14,276 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:13:14,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/1d3b7102ec624db196f821aa39dd5f02 is 50, key is test_row_0/B:col10/1695967993820/Put/seqid=0 2023-09-29T06:13:14,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-09-29T06:13:14,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=23 2023-09-29T06:13:14,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:14,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. as already flushing 2023-09-29T06:13:14,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:14,436 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] handler.RSProcedureHandler(53): pid=23 java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:14,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=23 java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:14,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=23 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:14,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:14,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968054509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:14,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:14,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968054509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:14,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:14,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968054515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:14,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-09-29T06:13:14,591 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=23 2023-09-29T06:13:14,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:14,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. as already flushing 2023-09-29T06:13:14,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:14,591 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=23}] handler.RSProcedureHandler(53): pid=23 java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:14,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=23 java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:14,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=23 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:14,730 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/1d3b7102ec624db196f821aa39dd5f02 2023-09-29T06:13:14,744 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/a8999a0b87a44794ac586a88ffb4b124 is 50, key is test_row_0/C:col10/1695967993820/Put/seqid=0 2023-09-29T06:13:14,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=23 2023-09-29T06:13:14,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:14,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. as already flushing 2023-09-29T06:13:14,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:14,751 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=23}] handler.RSProcedureHandler(53): pid=23 java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:14,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=23 java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:14,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=23 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:14,764 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/a8999a0b87a44794ac586a88ffb4b124 2023-09-29T06:13:14,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/a38030f69f774fdabc0c907151a01784 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/a38030f69f774fdabc0c907151a01784 2023-09-29T06:13:14,789 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/a38030f69f774fdabc0c907151a01784, entries=150, sequenceid=224, filesize=11.8 K 2023-09-29T06:13:14,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/1d3b7102ec624db196f821aa39dd5f02 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/1d3b7102ec624db196f821aa39dd5f02 2023-09-29T06:13:14,811 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/1d3b7102ec624db196f821aa39dd5f02, entries=150, sequenceid=224, filesize=11.8 K 2023-09-29T06:13:14,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/a8999a0b87a44794ac586a88ffb4b124 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/a8999a0b87a44794ac586a88ffb4b124 2023-09-29T06:13:14,820 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/a8999a0b87a44794ac586a88ffb4b124, entries=150, sequenceid=224, filesize=11.8 K 2023-09-29T06:13:14,821 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 1291e331f06575ebce0d38fd8ca0d9de in 1000ms, sequenceid=224, compaction requested=true 2023-09-29T06:13:14,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:14,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1291e331f06575ebce0d38fd8ca0d9de:A, priority=-2147483648, current under compaction store size is 1 2023-09-29T06:13:14,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:14,822 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:13:14,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1291e331f06575ebce0d38fd8ca0d9de:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:13:14,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:14,822 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:13:14,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1291e331f06575ebce0d38fd8ca0d9de:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:13:14,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:13:14,823 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36846 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:13:14,824 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36846 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:13:14,824 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 1291e331f06575ebce0d38fd8ca0d9de/A is initiating minor compaction (all files) 2023-09-29T06:13:14,824 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 1291e331f06575ebce0d38fd8ca0d9de/B is initiating minor compaction (all files) 2023-09-29T06:13:14,824 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1291e331f06575ebce0d38fd8ca0d9de/A in TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:14,824 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1291e331f06575ebce0d38fd8ca0d9de/B in TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:14,824 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/030789cb8ca74e25b544c375b0dc3038, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/e1d644c0965f4fa9a54ded5b8da40c0d, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/a38030f69f774fdabc0c907151a01784] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp, totalSize=36.0 K 2023-09-29T06:13:14,824 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/2d8a1971ac6848489cf4366da08a4bc8, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/3ab427fd69d5438fb058501d343cdbfb, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/1d3b7102ec624db196f821aa39dd5f02] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp, totalSize=36.0 K 2023-09-29T06:13:14,825 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 030789cb8ca74e25b544c375b0dc3038, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1695967991979 2023-09-29T06:13:14,825 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 2d8a1971ac6848489cf4366da08a4bc8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1695967991979 2023-09-29T06:13:14,825 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting e1d644c0965f4fa9a54ded5b8da40c0d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1695967992006 2023-09-29T06:13:14,826 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 3ab427fd69d5438fb058501d343cdbfb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1695967992006 2023-09-29T06:13:14,826 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting a38030f69f774fdabc0c907151a01784, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1695967992696 2023-09-29T06:13:14,826 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 1d3b7102ec624db196f821aa39dd5f02, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1695967992696 2023-09-29T06:13:14,840 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1291e331f06575ebce0d38fd8ca0d9de#A#compaction#48 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:14,841 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/cc83e59996bb458f80c6fb040c6ac977 is 50, key is test_row_0/A:col10/1695967993820/Put/seqid=0 2023-09-29T06:13:14,843 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1291e331f06575ebce0d38fd8ca0d9de#B#compaction#49 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:14,844 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/0eea780c66614ebb9ca0e00bee66463b is 50, key is test_row_0/B:col10/1695967993820/Put/seqid=0 2023-09-29T06:13:14,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-09-29T06:13:14,911 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=23 2023-09-29T06:13:14,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:14,912 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HRegion(2819): Flushing 1291e331f06575ebce0d38fd8ca0d9de 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2023-09-29T06:13:14,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=A 2023-09-29T06:13:14,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:14,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=B 2023-09-29T06:13:14,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:14,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=C 2023-09-29T06:13:14,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:14,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/7bbfa799a034448eb77e88ec394c0b97 is 50, key is test_row_0/A:col10/1695967993897/Put/seqid=0 2023-09-29T06:13:14,933 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/7bbfa799a034448eb77e88ec394c0b97 2023-09-29T06:13:14,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/42aa62ce42534ddd84a4d299ef8dcf79 is 50, key is test_row_0/B:col10/1695967993897/Put/seqid=0 2023-09-29T06:13:14,966 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/42aa62ce42534ddd84a4d299ef8dcf79 2023-09-29T06:13:14,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/e190d90f852d49928d8995003b6ed7e9 is 50, key is test_row_0/C:col10/1695967993897/Put/seqid=0 2023-09-29T06:13:15,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 1291e331f06575ebce0d38fd8ca0d9de 2023-09-29T06:13:15,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. as already flushing 2023-09-29T06:13:15,015 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/e190d90f852d49928d8995003b6ed7e9 2023-09-29T06:13:15,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/7bbfa799a034448eb77e88ec394c0b97 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/7bbfa799a034448eb77e88ec394c0b97 2023-09-29T06:13:15,032 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/7bbfa799a034448eb77e88ec394c0b97, entries=150, sequenceid=250, filesize=11.8 K 2023-09-29T06:13:15,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/42aa62ce42534ddd84a4d299ef8dcf79 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/42aa62ce42534ddd84a4d299ef8dcf79 2023-09-29T06:13:15,040 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/42aa62ce42534ddd84a4d299ef8dcf79, entries=150, sequenceid=250, filesize=11.8 K 2023-09-29T06:13:15,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/e190d90f852d49928d8995003b6ed7e9 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/e190d90f852d49928d8995003b6ed7e9 2023-09-29T06:13:15,052 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/e190d90f852d49928d8995003b6ed7e9, entries=150, sequenceid=250, filesize=11.8 K 2023-09-29T06:13:15,071 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HRegion(3022): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=26.84 KB/27480 for 1291e331f06575ebce0d38fd8ca0d9de in 159ms, sequenceid=250, compaction requested=true 2023-09-29T06:13:15,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HRegion(2520): Flush status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:15,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:15,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=23 2023-09-29T06:13:15,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=23 2023-09-29T06:13:15,084 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=23, resume processing ppid=22 2023-09-29T06:13:15,084 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=23, ppid=22, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 805 msec 2023-09-29T06:13:15,086 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=22, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=22, table=TestAcidGuarantees in 819 msec 2023-09-29T06:13:15,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 1291e331f06575ebce0d38fd8ca0d9de 2023-09-29T06:13:15,093 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 1291e331f06575ebce0d38fd8ca0d9de 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2023-09-29T06:13:15,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=A 2023-09-29T06:13:15,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:15,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=B 2023-09-29T06:13:15,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:15,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=C 2023-09-29T06:13:15,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:15,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/4835b823b1f44749952d47bb657b1fb4 is 50, key is test_row_1/A:col10/1695967995093/Put/seqid=0 2023-09-29T06:13:15,136 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/4835b823b1f44749952d47bb657b1fb4 2023-09-29T06:13:15,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:15,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968055139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:15,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:15,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968055142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:15,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:15,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968055139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:15,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/113365189067452ead856109076399c0 is 50, key is test_row_1/B:col10/1695967995093/Put/seqid=0 2023-09-29T06:13:15,164 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/113365189067452ead856109076399c0 2023-09-29T06:13:15,176 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/8955663353cf4bcfa6511397436d9731 is 50, key is test_row_1/C:col10/1695967995093/Put/seqid=0 2023-09-29T06:13:15,217 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/8955663353cf4bcfa6511397436d9731 2023-09-29T06:13:15,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/4835b823b1f44749952d47bb657b1fb4 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/4835b823b1f44749952d47bb657b1fb4 2023-09-29T06:13:15,235 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/4835b823b1f44749952d47bb657b1fb4, entries=200, sequenceid=262, filesize=14.3 K 2023-09-29T06:13:15,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/113365189067452ead856109076399c0 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/113365189067452ead856109076399c0 2023-09-29T06:13:15,245 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/113365189067452ead856109076399c0, entries=100, sequenceid=262, filesize=9.6 K 2023-09-29T06:13:15,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/8955663353cf4bcfa6511397436d9731 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/8955663353cf4bcfa6511397436d9731 2023-09-29T06:13:15,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:15,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968055246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:15,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:15,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968055247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:15,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:15,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968055247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:15,260 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/8955663353cf4bcfa6511397436d9731, entries=100, sequenceid=262, filesize=9.6 K 2023-09-29T06:13:15,261 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 1291e331f06575ebce0d38fd8ca0d9de in 169ms, sequenceid=262, compaction requested=true 2023-09-29T06:13:15,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:15,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1291e331f06575ebce0d38fd8ca0d9de:A, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:13:15,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2023-09-29T06:13:15,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1291e331f06575ebce0d38fd8ca0d9de:B, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:13:15,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2023-09-29T06:13:15,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1291e331f06575ebce0d38fd8ca0d9de:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:13:15,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2023-09-29T06:13:15,266 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/cc83e59996bb458f80c6fb040c6ac977 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/cc83e59996bb458f80c6fb040c6ac977 2023-09-29T06:13:15,274 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/0eea780c66614ebb9ca0e00bee66463b as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/0eea780c66614ebb9ca0e00bee66463b 2023-09-29T06:13:15,275 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 1291e331f06575ebce0d38fd8ca0d9de/A of 1291e331f06575ebce0d38fd8ca0d9de into cc83e59996bb458f80c6fb040c6ac977(size=12.4 K), total size for store is 38.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:15,275 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:15,275 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de., storeName=1291e331f06575ebce0d38fd8ca0d9de/A, priority=13, startTime=1695967994821; duration=0sec 2023-09-29T06:13:15,275 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2023-09-29T06:13:15,276 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1291e331f06575ebce0d38fd8ca0d9de:A 2023-09-29T06:13:15,276 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1291e331f06575ebce0d38fd8ca0d9de:A 2023-09-29T06:13:15,276 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2023-09-29T06:13:15,279 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 58820 starting at candidate #0 after considering 6 permutations with 6 in ratio 2023-09-29T06:13:15,279 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 1291e331f06575ebce0d38fd8ca0d9de/C is initiating minor compaction (all files) 2023-09-29T06:13:15,279 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1291e331f06575ebce0d38fd8ca0d9de/C in TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:15,279 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/e0105ac4698844b4a818e17907d613b1, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/3078da1ea1684bd48536ca94fc6500f8, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/a8999a0b87a44794ac586a88ffb4b124, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/e190d90f852d49928d8995003b6ed7e9, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/8955663353cf4bcfa6511397436d9731] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp, totalSize=57.4 K 2023-09-29T06:13:15,280 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting e0105ac4698844b4a818e17907d613b1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1695967991979 2023-09-29T06:13:15,281 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 3078da1ea1684bd48536ca94fc6500f8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1695967992006 2023-09-29T06:13:15,282 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting a8999a0b87a44794ac586a88ffb4b124, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1695967992696 2023-09-29T06:13:15,282 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 1291e331f06575ebce0d38fd8ca0d9de/B of 1291e331f06575ebce0d38fd8ca0d9de into 0eea780c66614ebb9ca0e00bee66463b(size=12.4 K), total size for store is 33.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:15,282 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:15,282 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de., storeName=1291e331f06575ebce0d38fd8ca0d9de/B, priority=13, startTime=1695967994822; duration=0sec 2023-09-29T06:13:15,282 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2023-09-29T06:13:15,282 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1291e331f06575ebce0d38fd8ca0d9de:B 2023-09-29T06:13:15,282 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1291e331f06575ebce0d38fd8ca0d9de:B 2023-09-29T06:13:15,282 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:13:15,283 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting e190d90f852d49928d8995003b6ed7e9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1695967993884 2023-09-29T06:13:15,284 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 8955663353cf4bcfa6511397436d9731, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1695967995038 2023-09-29T06:13:15,284 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39484 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:13:15,284 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 1291e331f06575ebce0d38fd8ca0d9de/A is initiating minor compaction (all files) 2023-09-29T06:13:15,284 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1291e331f06575ebce0d38fd8ca0d9de/A in TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:15,285 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/cc83e59996bb458f80c6fb040c6ac977, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/7bbfa799a034448eb77e88ec394c0b97, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/4835b823b1f44749952d47bb657b1fb4] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp, totalSize=38.6 K 2023-09-29T06:13:15,286 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting cc83e59996bb458f80c6fb040c6ac977, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1695967992696 2023-09-29T06:13:15,287 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 7bbfa799a034448eb77e88ec394c0b97, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1695967993884 2023-09-29T06:13:15,288 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 4835b823b1f44749952d47bb657b1fb4, keycount=200, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1695967995026 2023-09-29T06:13:15,307 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1291e331f06575ebce0d38fd8ca0d9de#A#compaction#56 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:15,307 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1291e331f06575ebce0d38fd8ca0d9de#C#compaction#57 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:15,308 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/f200b728a9134a1fbeeb50a6391aeaa1 is 50, key is test_row_0/C:col10/1695967993897/Put/seqid=0 2023-09-29T06:13:15,308 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/06356818e5d9423c9be67b38af6c83a4 is 50, key is test_row_0/A:col10/1695967993897/Put/seqid=0 2023-09-29T06:13:15,336 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/06356818e5d9423c9be67b38af6c83a4 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/06356818e5d9423c9be67b38af6c83a4 2023-09-29T06:13:15,343 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 1291e331f06575ebce0d38fd8ca0d9de/A of 1291e331f06575ebce0d38fd8ca0d9de into 06356818e5d9423c9be67b38af6c83a4(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:15,344 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:15,344 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de., storeName=1291e331f06575ebce0d38fd8ca0d9de/A, priority=13, startTime=1695967995262; duration=0sec 2023-09-29T06:13:15,344 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2023-09-29T06:13:15,344 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1291e331f06575ebce0d38fd8ca0d9de:A 2023-09-29T06:13:15,344 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:13:15,346 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34654 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:13:15,346 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 1291e331f06575ebce0d38fd8ca0d9de/B is initiating minor compaction (all files) 2023-09-29T06:13:15,346 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1291e331f06575ebce0d38fd8ca0d9de/B in TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:15,346 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/0eea780c66614ebb9ca0e00bee66463b, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/42aa62ce42534ddd84a4d299ef8dcf79, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/113365189067452ead856109076399c0] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp, totalSize=33.8 K 2023-09-29T06:13:15,347 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 0eea780c66614ebb9ca0e00bee66463b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1695967992696 2023-09-29T06:13:15,347 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 42aa62ce42534ddd84a4d299ef8dcf79, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1695967993884 2023-09-29T06:13:15,348 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 113365189067452ead856109076399c0, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1695967995038 2023-09-29T06:13:15,362 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1291e331f06575ebce0d38fd8ca0d9de#B#compaction#58 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:15,363 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/69a060142b6a42ddb01e8e01dd514e20 is 50, key is test_row_0/B:col10/1695967993897/Put/seqid=0 2023-09-29T06:13:15,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-09-29T06:13:15,380 INFO [Thread-152 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 22 completed 2023-09-29T06:13:15,383 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:13:15,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=24, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=24, table=TestAcidGuarantees 2023-09-29T06:13:15,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=24 2023-09-29T06:13:15,389 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/69a060142b6a42ddb01e8e01dd514e20 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/69a060142b6a42ddb01e8e01dd514e20 2023-09-29T06:13:15,391 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=24, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=24, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:13:15,392 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=24, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=24, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:13:15,392 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:13:15,399 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 1291e331f06575ebce0d38fd8ca0d9de/B of 1291e331f06575ebce0d38fd8ca0d9de into 69a060142b6a42ddb01e8e01dd514e20(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:15,399 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:15,399 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de., storeName=1291e331f06575ebce0d38fd8ca0d9de/B, priority=13, startTime=1695967995262; duration=0sec 2023-09-29T06:13:15,399 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:13:15,399 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1291e331f06575ebce0d38fd8ca0d9de:B 2023-09-29T06:13:15,400 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 5 compacting, 0 eligible, 16 blocking 2023-09-29T06:13:15,400 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2023-09-29T06:13:15,400 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2023-09-29T06:13:15,400 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. because compaction request was cancelled 2023-09-29T06:13:15,400 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1291e331f06575ebce0d38fd8ca0d9de:C 2023-09-29T06:13:15,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 1291e331f06575ebce0d38fd8ca0d9de 2023-09-29T06:13:15,456 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 1291e331f06575ebce0d38fd8ca0d9de 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2023-09-29T06:13:15,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=A 2023-09-29T06:13:15,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:15,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=B 2023-09-29T06:13:15,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:15,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=C 2023-09-29T06:13:15,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:15,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:15,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968055468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:15,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:15,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968055469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:15,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:15,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968055469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:15,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/38bdaa14502c46c2a7e24294978e32dd is 50, key is test_row_0/A:col10/1695967995455/Put/seqid=0 2023-09-29T06:13:15,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=24 2023-09-29T06:13:15,539 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/38bdaa14502c46c2a7e24294978e32dd 2023-09-29T06:13:15,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=25 2023-09-29T06:13:15,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:15,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. as already flushing 2023-09-29T06:13:15,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:15,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] handler.RSProcedureHandler(53): pid=25 java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:15,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=25 java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:15,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=25 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:15,557 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/38c54a91d4e34443a1bc6a66b721a4b6 is 50, key is test_row_0/B:col10/1695967995455/Put/seqid=0 2023-09-29T06:13:15,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:15,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968055574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:15,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:15,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968055574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:15,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:15,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968055575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:15,592 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/38c54a91d4e34443a1bc6a66b721a4b6 2023-09-29T06:13:15,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/1de1b459300b418ba856a997e70466c2 is 50, key is test_row_0/C:col10/1695967995455/Put/seqid=0 2023-09-29T06:13:15,613 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/1de1b459300b418ba856a997e70466c2 2023-09-29T06:13:15,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/38bdaa14502c46c2a7e24294978e32dd as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/38bdaa14502c46c2a7e24294978e32dd 2023-09-29T06:13:15,629 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/38bdaa14502c46c2a7e24294978e32dd, entries=200, sequenceid=293, filesize=14.4 K 2023-09-29T06:13:15,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/38c54a91d4e34443a1bc6a66b721a4b6 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/38c54a91d4e34443a1bc6a66b721a4b6 2023-09-29T06:13:15,636 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/38c54a91d4e34443a1bc6a66b721a4b6, entries=150, sequenceid=293, filesize=12.0 K 2023-09-29T06:13:15,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/1de1b459300b418ba856a997e70466c2 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/1de1b459300b418ba856a997e70466c2 2023-09-29T06:13:15,644 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/1de1b459300b418ba856a997e70466c2, entries=150, sequenceid=293, filesize=12.0 K 2023-09-29T06:13:15,645 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 1291e331f06575ebce0d38fd8ca0d9de in 189ms, sequenceid=293, compaction requested=false 2023-09-29T06:13:15,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:15,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=24 2023-09-29T06:13:15,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=25 2023-09-29T06:13:15,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:15,716 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HRegion(2819): Flushing 1291e331f06575ebce0d38fd8ca0d9de 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2023-09-29T06:13:15,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=A 2023-09-29T06:13:15,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:15,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=B 2023-09-29T06:13:15,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:15,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=C 2023-09-29T06:13:15,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:15,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/abe22ae6e20c41d1b298113861429f37 is 50, key is test_row_0/A:col10/1695967995465/Put/seqid=0 2023-09-29T06:13:15,741 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/abe22ae6e20c41d1b298113861429f37 2023-09-29T06:13:15,747 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/f200b728a9134a1fbeeb50a6391aeaa1 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/f200b728a9134a1fbeeb50a6391aeaa1 2023-09-29T06:13:15,755 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 5 (all) file(s) in 1291e331f06575ebce0d38fd8ca0d9de/C of 1291e331f06575ebce0d38fd8ca0d9de into f200b728a9134a1fbeeb50a6391aeaa1(size=12.5 K), total size for store is 24.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:15,755 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:15,755 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de., storeName=1291e331f06575ebce0d38fd8ca0d9de/C, priority=11, startTime=1695967995262; duration=0sec 2023-09-29T06:13:15,756 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:15,756 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1291e331f06575ebce0d38fd8ca0d9de:C 2023-09-29T06:13:15,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/8c6be74f0d9542a89050871cdf486be5 is 50, key is test_row_0/B:col10/1695967995465/Put/seqid=0 2023-09-29T06:13:15,771 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/8c6be74f0d9542a89050871cdf486be5 2023-09-29T06:13:15,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/f6393955edaf43de80a7b07f64bb19f3 is 50, key is test_row_0/C:col10/1695967995465/Put/seqid=0 2023-09-29T06:13:15,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. as already flushing 2023-09-29T06:13:15,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 1291e331f06575ebce0d38fd8ca0d9de 2023-09-29T06:13:15,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:15,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968055840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:15,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:15,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968055841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:15,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:15,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968055842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:15,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:15,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968055945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:15,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:15,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968055946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:15,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:15,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968055950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:15,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=24 2023-09-29T06:13:16,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:16,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968056148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:16,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:16,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968056148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:16,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:16,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968056152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:16,192 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/f6393955edaf43de80a7b07f64bb19f3 2023-09-29T06:13:16,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/abe22ae6e20c41d1b298113861429f37 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/abe22ae6e20c41d1b298113861429f37 2023-09-29T06:13:16,212 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/abe22ae6e20c41d1b298113861429f37, entries=150, sequenceid=302, filesize=12.0 K 2023-09-29T06:13:16,213 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xf8fdf8950a6e27e2: from storage DS-2e154dbb-27b5-42a6-ae9b-87cdc4287cdf node DatanodeRegistration(127.0.0.1:42427, datanodeUuid=3d6298e7-fdaf-4ed5-963c-adc3ebb4292e, infoPort=42279, infoSecurePort=0, ipcPort=42843, storageInfo=lv=-57;cid=testClusterID;nsid=45335293;c=1695967972816), blocks: 39, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2023-09-29T06:13:16,215 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xf8fdf8950a6e27e2: from storage DS-29ea5562-60e4-4bbb-b8e7-eac08b9465a9 node DatanodeRegistration(127.0.0.1:42427, datanodeUuid=3d6298e7-fdaf-4ed5-963c-adc3ebb4292e, infoPort=42279, infoSecurePort=0, ipcPort=42843, storageInfo=lv=-57;cid=testClusterID;nsid=45335293;c=1695967972816), blocks: 40, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-09-29T06:13:16,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/8c6be74f0d9542a89050871cdf486be5 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/8c6be74f0d9542a89050871cdf486be5 2023-09-29T06:13:16,226 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/8c6be74f0d9542a89050871cdf486be5, entries=150, sequenceid=302, filesize=12.0 K 2023-09-29T06:13:16,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/f6393955edaf43de80a7b07f64bb19f3 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/f6393955edaf43de80a7b07f64bb19f3 2023-09-29T06:13:16,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:16,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54718 deadline: 1695968056231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:16,236 DEBUG [Thread-142 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4158 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_0' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de., hostname=jenkins-hbase3.apache.org,39783,1695967978138, seqNum=2, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:572) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:559) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:197) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:218) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:750) 2023-09-29T06:13:16,239 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/f6393955edaf43de80a7b07f64bb19f3, entries=150, sequenceid=302, filesize=12.0 K 2023-09-29T06:13:16,240 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HRegion(3022): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for 1291e331f06575ebce0d38fd8ca0d9de in 524ms, sequenceid=302, compaction requested=true 2023-09-29T06:13:16,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HRegion(2520): Flush status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:16,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:16,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=25 2023-09-29T06:13:16,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=25 2023-09-29T06:13:16,250 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=25, resume processing ppid=24 2023-09-29T06:13:16,250 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=25, ppid=24, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 856 msec 2023-09-29T06:13:16,252 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=24, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=24, table=TestAcidGuarantees in 868 msec 2023-09-29T06:13:16,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 1291e331f06575ebce0d38fd8ca0d9de 2023-09-29T06:13:16,261 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 1291e331f06575ebce0d38fd8ca0d9de 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2023-09-29T06:13:16,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=A 2023-09-29T06:13:16,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:16,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=B 2023-09-29T06:13:16,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:16,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=C 2023-09-29T06:13:16,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:16,267 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/3ea27524b4274891a6cd847f529cfa76 is 50, key is test_row_0/A:col10/1695967995840/Put/seqid=0 2023-09-29T06:13:16,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:16,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968056276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:16,287 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/3ea27524b4274891a6cd847f529cfa76 2023-09-29T06:13:16,307 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/2c8dc76ec4ec44379bfa8fd4cdcf682c is 50, key is test_row_0/B:col10/1695967995840/Put/seqid=0 2023-09-29T06:13:16,320 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/2c8dc76ec4ec44379bfa8fd4cdcf682c 2023-09-29T06:13:16,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/7deb69431536494c9c7d98896ae8d456 is 50, key is test_row_0/C:col10/1695967995840/Put/seqid=0 2023-09-29T06:13:16,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:16,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968056378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:16,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:16,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968056451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:16,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:16,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968056453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:16,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:16,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968056462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:16,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=24 2023-09-29T06:13:16,495 INFO [Thread-152 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 24 completed 2023-09-29T06:13:16,497 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:13:16,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=26, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=26, table=TestAcidGuarantees 2023-09-29T06:13:16,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-09-29T06:13:16,501 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=26, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=26, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:13:16,502 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=26, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=26, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:13:16,502 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:13:16,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:16,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968056582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:16,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-09-29T06:13:16,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=27 2023-09-29T06:13:16,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:16,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. as already flushing 2023-09-29T06:13:16,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:16,662 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] handler.RSProcedureHandler(53): pid=27 java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:16,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=27 java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:16,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=27 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:16,739 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/7deb69431536494c9c7d98896ae8d456 2023-09-29T06:13:16,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/3ea27524b4274891a6cd847f529cfa76 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/3ea27524b4274891a6cd847f529cfa76 2023-09-29T06:13:16,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/3ea27524b4274891a6cd847f529cfa76, entries=200, sequenceid=331, filesize=14.4 K 2023-09-29T06:13:16,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/2c8dc76ec4ec44379bfa8fd4cdcf682c as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/2c8dc76ec4ec44379bfa8fd4cdcf682c 2023-09-29T06:13:16,779 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/2c8dc76ec4ec44379bfa8fd4cdcf682c, entries=150, sequenceid=331, filesize=12.0 K 2023-09-29T06:13:16,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/7deb69431536494c9c7d98896ae8d456 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/7deb69431536494c9c7d98896ae8d456 2023-09-29T06:13:16,789 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/7deb69431536494c9c7d98896ae8d456, entries=150, sequenceid=331, filesize=12.0 K 2023-09-29T06:13:16,791 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 1291e331f06575ebce0d38fd8ca0d9de in 530ms, sequenceid=331, compaction requested=true 2023-09-29T06:13:16,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:16,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1291e331f06575ebce0d38fd8ca0d9de:A, priority=-2147483648, current under compaction store size is 1 2023-09-29T06:13:16,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:16,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1291e331f06575ebce0d38fd8ca0d9de:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:13:16,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:13:16,791 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-29T06:13:16,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1291e331f06575ebce0d38fd8ca0d9de:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:13:16,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2023-09-29T06:13:16,791 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-29T06:13:16,793 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54614 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-29T06:13:16,794 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 1291e331f06575ebce0d38fd8ca0d9de/A is initiating minor compaction (all files) 2023-09-29T06:13:16,794 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1291e331f06575ebce0d38fd8ca0d9de/A in TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:16,794 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/06356818e5d9423c9be67b38af6c83a4, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/38bdaa14502c46c2a7e24294978e32dd, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/abe22ae6e20c41d1b298113861429f37, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/3ea27524b4274891a6cd847f529cfa76] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp, totalSize=53.3 K 2023-09-29T06:13:16,794 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49734 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-29T06:13:16,795 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 06356818e5d9423c9be67b38af6c83a4, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1695967993897 2023-09-29T06:13:16,795 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 1291e331f06575ebce0d38fd8ca0d9de/B is initiating minor compaction (all files) 2023-09-29T06:13:16,795 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1291e331f06575ebce0d38fd8ca0d9de/B in TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:16,795 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/69a060142b6a42ddb01e8e01dd514e20, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/38c54a91d4e34443a1bc6a66b721a4b6, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/8c6be74f0d9542a89050871cdf486be5, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/2c8dc76ec4ec44379bfa8fd4cdcf682c] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp, totalSize=48.6 K 2023-09-29T06:13:16,795 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 38bdaa14502c46c2a7e24294978e32dd, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1695967995130 2023-09-29T06:13:16,795 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 69a060142b6a42ddb01e8e01dd514e20, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1695967993897 2023-09-29T06:13:16,796 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting abe22ae6e20c41d1b298113861429f37, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1695967995460 2023-09-29T06:13:16,796 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 38c54a91d4e34443a1bc6a66b721a4b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1695967995454 2023-09-29T06:13:16,796 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 3ea27524b4274891a6cd847f529cfa76, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1695967995819 2023-09-29T06:13:16,796 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 8c6be74f0d9542a89050871cdf486be5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1695967995460 2023-09-29T06:13:16,797 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 2c8dc76ec4ec44379bfa8fd4cdcf682c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1695967995819 2023-09-29T06:13:16,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-09-29T06:13:16,809 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1291e331f06575ebce0d38fd8ca0d9de#A#compaction#68 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:16,810 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/7b17438bba3f4693904ac00e9a5e871b is 50, key is test_row_0/A:col10/1695967995840/Put/seqid=0 2023-09-29T06:13:16,810 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1291e331f06575ebce0d38fd8ca0d9de#B#compaction#69 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:16,811 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/74085fa665df4af88d24881738b174be is 50, key is test_row_0/B:col10/1695967995840/Put/seqid=0 2023-09-29T06:13:16,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=27 2023-09-29T06:13:16,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:16,819 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HRegion(2819): Flushing 1291e331f06575ebce0d38fd8ca0d9de 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2023-09-29T06:13:16,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=A 2023-09-29T06:13:16,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:16,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=B 2023-09-29T06:13:16,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:16,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=C 2023-09-29T06:13:16,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:16,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/05ec72ccf0cf4941a48fa91cc3b67cf6 is 50, key is test_row_0/A:col10/1695967996264/Put/seqid=0 2023-09-29T06:13:16,872 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/05ec72ccf0cf4941a48fa91cc3b67cf6 2023-09-29T06:13:16,875 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/74085fa665df4af88d24881738b174be as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/74085fa665df4af88d24881738b174be 2023-09-29T06:13:16,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/017af24239a4480ca217dfc927b84e4f is 50, key is test_row_0/B:col10/1695967996264/Put/seqid=0 2023-09-29T06:13:16,888 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 1291e331f06575ebce0d38fd8ca0d9de/B of 1291e331f06575ebce0d38fd8ca0d9de into 74085fa665df4af88d24881738b174be(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:16,888 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:16,888 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de., storeName=1291e331f06575ebce0d38fd8ca0d9de/B, priority=12, startTime=1695967996791; duration=0sec 2023-09-29T06:13:16,888 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:13:16,888 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1291e331f06575ebce0d38fd8ca0d9de:B 2023-09-29T06:13:16,888 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-29T06:13:16,893 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49700 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-29T06:13:16,893 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 1291e331f06575ebce0d38fd8ca0d9de/C is initiating minor compaction (all files) 2023-09-29T06:13:16,893 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1291e331f06575ebce0d38fd8ca0d9de/C in TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:16,893 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/f200b728a9134a1fbeeb50a6391aeaa1, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/1de1b459300b418ba856a997e70466c2, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/f6393955edaf43de80a7b07f64bb19f3, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/7deb69431536494c9c7d98896ae8d456] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp, totalSize=48.5 K 2023-09-29T06:13:16,895 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting f200b728a9134a1fbeeb50a6391aeaa1, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1695967993897 2023-09-29T06:13:16,896 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 1de1b459300b418ba856a997e70466c2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1695967995454 2023-09-29T06:13:16,896 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting f6393955edaf43de80a7b07f64bb19f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1695967995460 2023-09-29T06:13:16,896 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 7deb69431536494c9c7d98896ae8d456, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1695967995819 2023-09-29T06:13:16,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. as already flushing 2023-09-29T06:13:16,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 1291e331f06575ebce0d38fd8ca0d9de 2023-09-29T06:13:16,906 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/017af24239a4480ca217dfc927b84e4f 2023-09-29T06:13:16,908 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1291e331f06575ebce0d38fd8ca0d9de#C#compaction#72 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:16,908 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/d65420780b02414a884196fb489e2792 is 50, key is test_row_0/C:col10/1695967995840/Put/seqid=0 2023-09-29T06:13:16,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/64c6e7035a744473ae2bc9406622c96b is 50, key is test_row_0/C:col10/1695967996264/Put/seqid=0 2023-09-29T06:13:16,987 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/d65420780b02414a884196fb489e2792 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/d65420780b02414a884196fb489e2792 2023-09-29T06:13:16,993 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/64c6e7035a744473ae2bc9406622c96b 2023-09-29T06:13:16,995 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 1291e331f06575ebce0d38fd8ca0d9de/C of 1291e331f06575ebce0d38fd8ca0d9de into d65420780b02414a884196fb489e2792(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:16,995 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:16,995 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de., storeName=1291e331f06575ebce0d38fd8ca0d9de/C, priority=12, startTime=1695967996791; duration=0sec 2023-09-29T06:13:16,995 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:16,995 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1291e331f06575ebce0d38fd8ca0d9de:C 2023-09-29T06:13:17,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/05ec72ccf0cf4941a48fa91cc3b67cf6 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/05ec72ccf0cf4941a48fa91cc3b67cf6 2023-09-29T06:13:17,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:17,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968057003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:17,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:17,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968057003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:17,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:17,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968056996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:17,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:17,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968057005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:17,011 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/05ec72ccf0cf4941a48fa91cc3b67cf6, entries=150, sequenceid=339, filesize=12.0 K 2023-09-29T06:13:17,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/017af24239a4480ca217dfc927b84e4f as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/017af24239a4480ca217dfc927b84e4f 2023-09-29T06:13:17,020 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/017af24239a4480ca217dfc927b84e4f, entries=150, sequenceid=339, filesize=12.0 K 2023-09-29T06:13:17,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/64c6e7035a744473ae2bc9406622c96b as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/64c6e7035a744473ae2bc9406622c96b 2023-09-29T06:13:17,027 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/64c6e7035a744473ae2bc9406622c96b, entries=150, sequenceid=339, filesize=12.0 K 2023-09-29T06:13:17,028 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HRegion(3022): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=174.43 KB/178620 for 1291e331f06575ebce0d38fd8ca0d9de in 209ms, sequenceid=339, compaction requested=false 2023-09-29T06:13:17,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HRegion(2520): Flush status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:17,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:17,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=27 2023-09-29T06:13:17,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=27 2023-09-29T06:13:17,037 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=27, resume processing ppid=26 2023-09-29T06:13:17,037 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=27, ppid=26, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 533 msec 2023-09-29T06:13:17,039 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=26, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=26, table=TestAcidGuarantees in 541 msec 2023-09-29T06:13:17,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 1291e331f06575ebce0d38fd8ca0d9de 2023-09-29T06:13:17,113 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 1291e331f06575ebce0d38fd8ca0d9de 3/3 column families, dataSize=187.85 KB heapSize=492.94 KB 2023-09-29T06:13:17,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=A 2023-09-29T06:13:17,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:17,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=B 2023-09-29T06:13:17,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:17,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=C 2023-09-29T06:13:17,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:17,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-09-29T06:13:17,119 INFO [Thread-152 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 26 completed 2023-09-29T06:13:17,122 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:13:17,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/3140377bcca74f89853bb266eb801def is 50, key is test_row_0/A:col10/1695967997110/Put/seqid=0 2023-09-29T06:13:17,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=28, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=28, table=TestAcidGuarantees 2023-09-29T06:13:17,127 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=28, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=28, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:13:17,128 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=28, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=28, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:13:17,128 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:13:17,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:17,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968057118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:17,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:17,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=28 2023-09-29T06:13:17,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968057121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:17,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:17,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968057129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:17,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:17,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968057129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:17,141 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/3140377bcca74f89853bb266eb801def 2023-09-29T06:13:17,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/3927ce7be2aa43fe9bc745376e11e66b is 50, key is test_row_0/B:col10/1695967997110/Put/seqid=0 2023-09-29T06:13:17,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:17,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968057230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:17,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=28 2023-09-29T06:13:17,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:17,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968057234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:17,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:17,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968057238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:17,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:17,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968057238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:17,266 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/7b17438bba3f4693904ac00e9a5e871b as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/7b17438bba3f4693904ac00e9a5e871b 2023-09-29T06:13:17,274 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 1291e331f06575ebce0d38fd8ca0d9de/A of 1291e331f06575ebce0d38fd8ca0d9de into 7b17438bba3f4693904ac00e9a5e871b(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:17,275 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:17,275 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de., storeName=1291e331f06575ebce0d38fd8ca0d9de/A, priority=12, startTime=1695967996791; duration=0sec 2023-09-29T06:13:17,275 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:17,275 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1291e331f06575ebce0d38fd8ca0d9de:A 2023-09-29T06:13:17,284 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=29 2023-09-29T06:13:17,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:17,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. as already flushing 2023-09-29T06:13:17,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:17,284 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] handler.RSProcedureHandler(53): pid=29 java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:17,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=29 java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:17,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=29 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:17,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=28 2023-09-29T06:13:17,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:17,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968057434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:17,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:17,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968057438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:17,439 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=29 2023-09-29T06:13:17,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:17,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. as already flushing 2023-09-29T06:13:17,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:17,440 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] handler.RSProcedureHandler(53): pid=29 java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:17,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=29 java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:17,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:17,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968057441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:17,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=29 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:17,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:17,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968057451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:17,564 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/3927ce7be2aa43fe9bc745376e11e66b 2023-09-29T06:13:17,582 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/268074203da74613ac5edb040b091e12 is 50, key is test_row_0/C:col10/1695967997110/Put/seqid=0 2023-09-29T06:13:17,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=29 2023-09-29T06:13:17,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:17,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. as already flushing 2023-09-29T06:13:17,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:17,597 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] handler.RSProcedureHandler(53): pid=29 java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:17,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=29 java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:17,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=29 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:17,599 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/268074203da74613ac5edb040b091e12 2023-09-29T06:13:17,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/3140377bcca74f89853bb266eb801def as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/3140377bcca74f89853bb266eb801def 2023-09-29T06:13:17,627 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/3140377bcca74f89853bb266eb801def, entries=150, sequenceid=372, filesize=12.0 K 2023-09-29T06:13:17,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/3927ce7be2aa43fe9bc745376e11e66b as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/3927ce7be2aa43fe9bc745376e11e66b 2023-09-29T06:13:17,643 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/3927ce7be2aa43fe9bc745376e11e66b, entries=150, sequenceid=372, filesize=12.0 K 2023-09-29T06:13:17,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/268074203da74613ac5edb040b091e12 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/268074203da74613ac5edb040b091e12 2023-09-29T06:13:17,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/268074203da74613ac5edb040b091e12, entries=150, sequenceid=372, filesize=12.0 K 2023-09-29T06:13:17,661 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~187.85 KB/192360, heapSize ~492.89 KB/504720, currentSize=20.13 KB/20610 for 1291e331f06575ebce0d38fd8ca0d9de in 548ms, sequenceid=372, compaction requested=true 2023-09-29T06:13:17,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:17,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1291e331f06575ebce0d38fd8ca0d9de:A, priority=-2147483648, current under compaction store size is 1 2023-09-29T06:13:17,662 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:13:17,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:17,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1291e331f06575ebce0d38fd8ca0d9de:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:13:17,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:17,662 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:13:17,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1291e331f06575ebce0d38fd8ca0d9de:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:13:17,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:13:17,669 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37636 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:13:17,669 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 1291e331f06575ebce0d38fd8ca0d9de/A is initiating minor compaction (all files) 2023-09-29T06:13:17,669 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37636 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:13:17,669 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 1291e331f06575ebce0d38fd8ca0d9de/B is initiating minor compaction (all files) 2023-09-29T06:13:17,669 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1291e331f06575ebce0d38fd8ca0d9de/A in TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:17,669 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1291e331f06575ebce0d38fd8ca0d9de/B in TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:17,669 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/7b17438bba3f4693904ac00e9a5e871b, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/05ec72ccf0cf4941a48fa91cc3b67cf6, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/3140377bcca74f89853bb266eb801def] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp, totalSize=36.8 K 2023-09-29T06:13:17,669 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/74085fa665df4af88d24881738b174be, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/017af24239a4480ca217dfc927b84e4f, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/3927ce7be2aa43fe9bc745376e11e66b] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp, totalSize=36.8 K 2023-09-29T06:13:17,671 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 7b17438bba3f4693904ac00e9a5e871b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1695967995819 2023-09-29T06:13:17,671 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 74085fa665df4af88d24881738b174be, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1695967995819 2023-09-29T06:13:17,671 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 017af24239a4480ca217dfc927b84e4f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1695967996264 2023-09-29T06:13:17,672 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 3927ce7be2aa43fe9bc745376e11e66b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1695967996983 2023-09-29T06:13:17,672 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 05ec72ccf0cf4941a48fa91cc3b67cf6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1695967996264 2023-09-29T06:13:17,673 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 3140377bcca74f89853bb266eb801def, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1695967996983 2023-09-29T06:13:17,694 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1291e331f06575ebce0d38fd8ca0d9de#A#compaction#77 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:17,695 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/9470ee2994744772b820295bd9f409d1 is 50, key is test_row_0/A:col10/1695967997110/Put/seqid=0 2023-09-29T06:13:17,695 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1291e331f06575ebce0d38fd8ca0d9de#B#compaction#78 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:17,696 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/e6def42d2a0f42cf940009c986febdb5 is 50, key is test_row_0/B:col10/1695967997110/Put/seqid=0 2023-09-29T06:13:17,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=28 2023-09-29T06:13:17,740 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/9470ee2994744772b820295bd9f409d1 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/9470ee2994744772b820295bd9f409d1 2023-09-29T06:13:17,747 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/e6def42d2a0f42cf940009c986febdb5 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/e6def42d2a0f42cf940009c986febdb5 2023-09-29T06:13:17,751 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 1291e331f06575ebce0d38fd8ca0d9de/A of 1291e331f06575ebce0d38fd8ca0d9de into 9470ee2994744772b820295bd9f409d1(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:17,751 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:17,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 1291e331f06575ebce0d38fd8ca0d9de 2023-09-29T06:13:17,751 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de., storeName=1291e331f06575ebce0d38fd8ca0d9de/A, priority=13, startTime=1695967997662; duration=0sec 2023-09-29T06:13:17,751 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 1291e331f06575ebce0d38fd8ca0d9de 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2023-09-29T06:13:17,751 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:13:17,751 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1291e331f06575ebce0d38fd8ca0d9de:A 2023-09-29T06:13:17,751 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:13:17,751 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=29 2023-09-29T06:13:17,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:17,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. as already flushing 2023-09-29T06:13:17,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:17,752 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] handler.RSProcedureHandler(53): pid=29 java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:17,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=29 java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:17,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=29 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:17,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=A 2023-09-29T06:13:17,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:17,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=B 2023-09-29T06:13:17,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:17,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=C 2023-09-29T06:13:17,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:17,756 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37602 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:13:17,756 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 1291e331f06575ebce0d38fd8ca0d9de/C is initiating minor compaction (all files) 2023-09-29T06:13:17,756 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1291e331f06575ebce0d38fd8ca0d9de/C in TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:17,756 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/d65420780b02414a884196fb489e2792, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/64c6e7035a744473ae2bc9406622c96b, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/268074203da74613ac5edb040b091e12] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp, totalSize=36.7 K 2023-09-29T06:13:17,758 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting d65420780b02414a884196fb489e2792, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1695967995819 2023-09-29T06:13:17,758 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 64c6e7035a744473ae2bc9406622c96b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1695967996264 2023-09-29T06:13:17,759 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 268074203da74613ac5edb040b091e12, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1695967996983 2023-09-29T06:13:17,761 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 1291e331f06575ebce0d38fd8ca0d9de/B of 1291e331f06575ebce0d38fd8ca0d9de into e6def42d2a0f42cf940009c986febdb5(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:17,761 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:17,761 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de., storeName=1291e331f06575ebce0d38fd8ca0d9de/B, priority=13, startTime=1695967997662; duration=0sec 2023-09-29T06:13:17,761 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:17,761 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1291e331f06575ebce0d38fd8ca0d9de:B 2023-09-29T06:13:17,762 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/6d31eb78e62b49ea86f6548379c26831 is 50, key is test_row_0/A:col10/1695967997750/Put/seqid=0 2023-09-29T06:13:17,786 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1291e331f06575ebce0d38fd8ca0d9de#C#compaction#80 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:17,787 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/7de828c345454202b934a75bbb6d2451 is 50, key is test_row_0/C:col10/1695967997110/Put/seqid=0 2023-09-29T06:13:17,802 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/6d31eb78e62b49ea86f6548379c26831 2023-09-29T06:13:17,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/02418b3fd8ce4cb28ce440a0a6503464 is 50, key is test_row_0/B:col10/1695967997750/Put/seqid=0 2023-09-29T06:13:17,831 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/02418b3fd8ce4cb28ce440a0a6503464 2023-09-29T06:13:17,840 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/fea8317613ec48d9a42058ef39d1b830 is 50, key is test_row_0/C:col10/1695967997750/Put/seqid=0 2023-09-29T06:13:17,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:17,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968057854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:17,859 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/fea8317613ec48d9a42058ef39d1b830 2023-09-29T06:13:17,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:17,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968057855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:17,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:17,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968057856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:17,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:17,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968057858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:17,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/6d31eb78e62b49ea86f6548379c26831 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/6d31eb78e62b49ea86f6548379c26831 2023-09-29T06:13:17,884 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/6d31eb78e62b49ea86f6548379c26831, entries=150, sequenceid=386, filesize=12.0 K 2023-09-29T06:13:17,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/02418b3fd8ce4cb28ce440a0a6503464 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/02418b3fd8ce4cb28ce440a0a6503464 2023-09-29T06:13:17,893 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/02418b3fd8ce4cb28ce440a0a6503464, entries=100, sequenceid=386, filesize=9.6 K 2023-09-29T06:13:17,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/fea8317613ec48d9a42058ef39d1b830 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/fea8317613ec48d9a42058ef39d1b830 2023-09-29T06:13:17,904 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/fea8317613ec48d9a42058ef39d1b830, entries=100, sequenceid=386, filesize=9.6 K 2023-09-29T06:13:17,905 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 1291e331f06575ebce0d38fd8ca0d9de in 154ms, sequenceid=386, compaction requested=false 2023-09-29T06:13:17,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:17,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=29 2023-09-29T06:13:17,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:17,908 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegion(2819): Flushing 1291e331f06575ebce0d38fd8ca0d9de 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2023-09-29T06:13:17,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=A 2023-09-29T06:13:17,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:17,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=B 2023-09-29T06:13:17,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:17,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=C 2023-09-29T06:13:17,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:17,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/e3f6d5929faf43bb81e01f70b9d8bb8b is 50, key is test_row_0/A:col10/1695967997837/Put/seqid=0 2023-09-29T06:13:17,928 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/e3f6d5929faf43bb81e01f70b9d8bb8b 2023-09-29T06:13:17,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/55cb5c428dfb44788fe0dece9a36a9a0 is 50, key is test_row_0/B:col10/1695967997837/Put/seqid=0 2023-09-29T06:13:17,961 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/55cb5c428dfb44788fe0dece9a36a9a0 2023-09-29T06:13:17,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 1291e331f06575ebce0d38fd8ca0d9de 2023-09-29T06:13:17,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. as already flushing 2023-09-29T06:13:17,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/7c09918d63564284a3d8885530d6b18f is 50, key is test_row_0/C:col10/1695967997837/Put/seqid=0 2023-09-29T06:13:17,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:17,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968057972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:17,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:17,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968057973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:17,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:17,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968057979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:17,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:17,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968057986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:18,019 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/7c09918d63564284a3d8885530d6b18f 2023-09-29T06:13:18,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/e3f6d5929faf43bb81e01f70b9d8bb8b as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/e3f6d5929faf43bb81e01f70b9d8bb8b 2023-09-29T06:13:18,035 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/e3f6d5929faf43bb81e01f70b9d8bb8b, entries=150, sequenceid=412, filesize=12.0 K 2023-09-29T06:13:18,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/55cb5c428dfb44788fe0dece9a36a9a0 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/55cb5c428dfb44788fe0dece9a36a9a0 2023-09-29T06:13:18,048 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/55cb5c428dfb44788fe0dece9a36a9a0, entries=150, sequenceid=412, filesize=12.0 K 2023-09-29T06:13:18,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/7c09918d63564284a3d8885530d6b18f as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/7c09918d63564284a3d8885530d6b18f 2023-09-29T06:13:18,057 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/7c09918d63564284a3d8885530d6b18f, entries=150, sequenceid=412, filesize=12.0 K 2023-09-29T06:13:18,059 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegion(3022): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 1291e331f06575ebce0d38fd8ca0d9de in 151ms, sequenceid=412, compaction requested=true 2023-09-29T06:13:18,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegion(2520): Flush status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:18,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:18,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=29 2023-09-29T06:13:18,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=29 2023-09-29T06:13:18,067 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=29, resume processing ppid=28 2023-09-29T06:13:18,067 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=29, ppid=28, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 936 msec 2023-09-29T06:13:18,069 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=28, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=28, table=TestAcidGuarantees in 946 msec 2023-09-29T06:13:18,081 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 1291e331f06575ebce0d38fd8ca0d9de 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2023-09-29T06:13:18,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=A 2023-09-29T06:13:18,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:18,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=B 2023-09-29T06:13:18,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:18,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=C 2023-09-29T06:13:18,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 1291e331f06575ebce0d38fd8ca0d9de 2023-09-29T06:13:18,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:18,091 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/4a852996bd5e4756b823e05ff93648b5 is 50, key is test_row_0/A:col10/1695967997970/Put/seqid=0 2023-09-29T06:13:18,116 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/4a852996bd5e4756b823e05ff93648b5 2023-09-29T06:13:18,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/da93a948772342edb8e93a6329cc7455 is 50, key is test_row_0/B:col10/1695967997970/Put/seqid=0 2023-09-29T06:13:18,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:18,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968058123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:18,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:18,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968058124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:18,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:18,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:18,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968058127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:18,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968058127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:18,141 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/da93a948772342edb8e93a6329cc7455 2023-09-29T06:13:18,155 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/b36fd3ea48e74b798497df97a19b0e90 is 50, key is test_row_0/C:col10/1695967997970/Put/seqid=0 2023-09-29T06:13:18,180 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/b36fd3ea48e74b798497df97a19b0e90 2023-09-29T06:13:18,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/4a852996bd5e4756b823e05ff93648b5 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/4a852996bd5e4756b823e05ff93648b5 2023-09-29T06:13:18,204 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/4a852996bd5e4756b823e05ff93648b5, entries=200, sequenceid=425, filesize=14.4 K 2023-09-29T06:13:18,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/da93a948772342edb8e93a6329cc7455 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/da93a948772342edb8e93a6329cc7455 2023-09-29T06:13:18,214 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/da93a948772342edb8e93a6329cc7455, entries=150, sequenceid=425, filesize=12.0 K 2023-09-29T06:13:18,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/b36fd3ea48e74b798497df97a19b0e90 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/b36fd3ea48e74b798497df97a19b0e90 2023-09-29T06:13:18,225 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/b36fd3ea48e74b798497df97a19b0e90, entries=150, sequenceid=425, filesize=12.0 K 2023-09-29T06:13:18,227 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 1291e331f06575ebce0d38fd8ca0d9de in 145ms, sequenceid=425, compaction requested=true 2023-09-29T06:13:18,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:18,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1291e331f06575ebce0d38fd8ca0d9de:A, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:13:18,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:18,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1291e331f06575ebce0d38fd8ca0d9de:B, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:13:18,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:13:18,227 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-29T06:13:18,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1291e331f06575ebce0d38fd8ca0d9de:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:13:18,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2023-09-29T06:13:18,228 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52458 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-29T06:13:18,228 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 1291e331f06575ebce0d38fd8ca0d9de/A is initiating minor compaction (all files) 2023-09-29T06:13:18,229 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1291e331f06575ebce0d38fd8ca0d9de/A in TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:18,229 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/9470ee2994744772b820295bd9f409d1, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/6d31eb78e62b49ea86f6548379c26831, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/e3f6d5929faf43bb81e01f70b9d8bb8b, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/4a852996bd5e4756b823e05ff93648b5] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp, totalSize=51.2 K 2023-09-29T06:13:18,229 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 9470ee2994744772b820295bd9f409d1, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1695967996983 2023-09-29T06:13:18,229 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 6d31eb78e62b49ea86f6548379c26831, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1695967997745 2023-09-29T06:13:18,230 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting e3f6d5929faf43bb81e01f70b9d8bb8b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1695967997837 2023-09-29T06:13:18,232 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 4a852996bd5e4756b823e05ff93648b5, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1695967997965 2023-09-29T06:13:18,233 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/7de828c345454202b934a75bbb6d2451 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/7de828c345454202b934a75bbb6d2451 2023-09-29T06:13:18,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=28 2023-09-29T06:13:18,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 1291e331f06575ebce0d38fd8ca0d9de 2023-09-29T06:13:18,237 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 1291e331f06575ebce0d38fd8ca0d9de 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2023-09-29T06:13:18,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=A 2023-09-29T06:13:18,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:18,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=B 2023-09-29T06:13:18,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:18,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=C 2023-09-29T06:13:18,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:18,238 INFO [Thread-152 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 28 completed 2023-09-29T06:13:18,248 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:13:18,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=30, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=30, table=TestAcidGuarantees 2023-09-29T06:13:18,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=30 2023-09-29T06:13:18,258 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=30, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=30, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:13:18,260 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/13d4415cb3a2450c901b16d417402092 is 50, key is test_row_0/A:col10/1695967998235/Put/seqid=0 2023-09-29T06:13:18,262 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=30, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=30, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:13:18,262 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:13:18,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:18,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968058252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:18,267 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 1291e331f06575ebce0d38fd8ca0d9de/C of 1291e331f06575ebce0d38fd8ca0d9de into 7de828c345454202b934a75bbb6d2451(size=12.8 K), total size for store is 46.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:18,267 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:18,267 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de., storeName=1291e331f06575ebce0d38fd8ca0d9de/C, priority=13, startTime=1695967997662; duration=0sec 2023-09-29T06:13:18,267 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2023-09-29T06:13:18,267 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1291e331f06575ebce0d38fd8ca0d9de:C 2023-09-29T06:13:18,267 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1291e331f06575ebce0d38fd8ca0d9de:C 2023-09-29T06:13:18,267 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-29T06:13:18,270 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47578 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-29T06:13:18,270 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 1291e331f06575ebce0d38fd8ca0d9de/B is initiating minor compaction (all files) 2023-09-29T06:13:18,271 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1291e331f06575ebce0d38fd8ca0d9de/B in TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:18,271 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/e6def42d2a0f42cf940009c986febdb5, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/02418b3fd8ce4cb28ce440a0a6503464, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/55cb5c428dfb44788fe0dece9a36a9a0, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/da93a948772342edb8e93a6329cc7455] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp, totalSize=46.5 K 2023-09-29T06:13:18,271 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting e6def42d2a0f42cf940009c986febdb5, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1695967996983 2023-09-29T06:13:18,273 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 02418b3fd8ce4cb28ce440a0a6503464, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1695967997748 2023-09-29T06:13:18,274 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 55cb5c428dfb44788fe0dece9a36a9a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1695967997837 2023-09-29T06:13:18,274 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting da93a948772342edb8e93a6329cc7455, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1695967997965 2023-09-29T06:13:18,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:18,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968058265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:18,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:18,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968058265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:18,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:18,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968058274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:18,287 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1291e331f06575ebce0d38fd8ca0d9de#A#compaction#90 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:18,287 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/aad8e39a39904d5cafdeeb0f4585d6cd is 50, key is test_row_0/A:col10/1695967997970/Put/seqid=0 2023-09-29T06:13:18,295 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1291e331f06575ebce0d38fd8ca0d9de#B#compaction#91 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:18,296 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/e79c5123683b4c57b9715dae4a062a26 is 50, key is test_row_0/B:col10/1695967997970/Put/seqid=0 2023-09-29T06:13:18,338 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/13d4415cb3a2450c901b16d417402092 2023-09-29T06:13:18,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=30 2023-09-29T06:13:18,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/f14a987c2aca43c4be9f2f45ad72cecb is 50, key is test_row_0/B:col10/1695967998235/Put/seqid=0 2023-09-29T06:13:18,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:18,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968058368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:18,375 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/f14a987c2aca43c4be9f2f45ad72cecb 2023-09-29T06:13:18,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:18,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968058380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:18,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:18,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968058380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:18,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:18,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968058383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:18,388 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/1bb3b1f26ea84029ba07aad4338133d8 is 50, key is test_row_0/C:col10/1695967998235/Put/seqid=0 2023-09-29T06:13:18,397 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/1bb3b1f26ea84029ba07aad4338133d8 2023-09-29T06:13:18,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/13d4415cb3a2450c901b16d417402092 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/13d4415cb3a2450c901b16d417402092 2023-09-29T06:13:18,411 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/13d4415cb3a2450c901b16d417402092, entries=250, sequenceid=451, filesize=16.8 K 2023-09-29T06:13:18,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/f14a987c2aca43c4be9f2f45ad72cecb as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/f14a987c2aca43c4be9f2f45ad72cecb 2023-09-29T06:13:18,416 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=31 2023-09-29T06:13:18,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:18,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. as already flushing 2023-09-29T06:13:18,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:18,417 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=31}] handler.RSProcedureHandler(53): pid=31 java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:18,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=31 java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:18,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=31 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:18,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/f14a987c2aca43c4be9f2f45ad72cecb, entries=150, sequenceid=451, filesize=12.0 K 2023-09-29T06:13:18,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/1bb3b1f26ea84029ba07aad4338133d8 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/1bb3b1f26ea84029ba07aad4338133d8 2023-09-29T06:13:18,473 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/1bb3b1f26ea84029ba07aad4338133d8, entries=150, sequenceid=451, filesize=12.0 K 2023-09-29T06:13:18,474 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 1291e331f06575ebce0d38fd8ca0d9de in 237ms, sequenceid=451, compaction requested=true 2023-09-29T06:13:18,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:18,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1291e331f06575ebce0d38fd8ca0d9de:A, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:13:18,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2023-09-29T06:13:18,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1291e331f06575ebce0d38fd8ca0d9de:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:13:18,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2023-09-29T06:13:18,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1291e331f06575ebce0d38fd8ca0d9de:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:13:18,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2023-09-29T06:13:18,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=30 2023-09-29T06:13:18,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 1291e331f06575ebce0d38fd8ca0d9de 2023-09-29T06:13:18,576 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 1291e331f06575ebce0d38fd8ca0d9de 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2023-09-29T06:13:18,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=A 2023-09-29T06:13:18,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:18,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=B 2023-09-29T06:13:18,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:18,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=C 2023-09-29T06:13:18,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:18,585 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/06b6f949408b4e95be3e89d4fb83b0a2 is 50, key is test_row_0/A:col10/1695967998263/Put/seqid=0 2023-09-29T06:13:18,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=31 2023-09-29T06:13:18,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:18,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. as already flushing 2023-09-29T06:13:18,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:18,586 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=31}] handler.RSProcedureHandler(53): pid=31 java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:18,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=31 java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:18,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=31 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:18,615 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/06b6f949408b4e95be3e89d4fb83b0a2 2023-09-29T06:13:18,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/31cf797c975749b0918f22b71066ee01 is 50, key is test_row_0/B:col10/1695967998263/Put/seqid=0 2023-09-29T06:13:18,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:18,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968058632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:18,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:18,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968058634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:18,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:18,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968058635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:18,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:18,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968058639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:18,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:18,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968058741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:18,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:18,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968058742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:18,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:18,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:18,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968058742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:18,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968058744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:18,750 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/e79c5123683b4c57b9715dae4a062a26 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/e79c5123683b4c57b9715dae4a062a26 2023-09-29T06:13:18,754 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=31 2023-09-29T06:13:18,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:18,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. as already flushing 2023-09-29T06:13:18,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:18,754 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] handler.RSProcedureHandler(53): pid=31 java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:18,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=31 java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:18,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=31 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:18,766 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/aad8e39a39904d5cafdeeb0f4585d6cd as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/aad8e39a39904d5cafdeeb0f4585d6cd 2023-09-29T06:13:18,770 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 1291e331f06575ebce0d38fd8ca0d9de/B of 1291e331f06575ebce0d38fd8ca0d9de into e79c5123683b4c57b9715dae4a062a26(size=13.0 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:18,770 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:18,770 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de., storeName=1291e331f06575ebce0d38fd8ca0d9de/B, priority=12, startTime=1695967998227; duration=0sec 2023-09-29T06:13:18,770 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2023-09-29T06:13:18,770 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1291e331f06575ebce0d38fd8ca0d9de:B 2023-09-29T06:13:18,771 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 4 compacting, 1 eligible, 16 blocking 2023-09-29T06:13:18,771 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2023-09-29T06:13:18,771 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2023-09-29T06:13:18,771 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. because compaction request was cancelled 2023-09-29T06:13:18,771 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1291e331f06575ebce0d38fd8ca0d9de:A 2023-09-29T06:13:18,771 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2023-09-29T06:13:18,774 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 59828 starting at candidate #0 after considering 6 permutations with 6 in ratio 2023-09-29T06:13:18,774 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 1291e331f06575ebce0d38fd8ca0d9de/C is initiating minor compaction (all files) 2023-09-29T06:13:18,774 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1291e331f06575ebce0d38fd8ca0d9de/C in TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:18,774 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 1291e331f06575ebce0d38fd8ca0d9de/A of 1291e331f06575ebce0d38fd8ca0d9de into aad8e39a39904d5cafdeeb0f4585d6cd(size=13.0 K), total size for store is 29.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:18,774 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:18,774 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/7de828c345454202b934a75bbb6d2451, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/fea8317613ec48d9a42058ef39d1b830, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/7c09918d63564284a3d8885530d6b18f, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/b36fd3ea48e74b798497df97a19b0e90, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/1bb3b1f26ea84029ba07aad4338133d8] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp, totalSize=58.4 K 2023-09-29T06:13:18,774 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de., storeName=1291e331f06575ebce0d38fd8ca0d9de/A, priority=12, startTime=1695967998227; duration=0sec 2023-09-29T06:13:18,774 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2023-09-29T06:13:18,774 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1291e331f06575ebce0d38fd8ca0d9de:A 2023-09-29T06:13:18,774 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1291e331f06575ebce0d38fd8ca0d9de:B 2023-09-29T06:13:18,775 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 5 compacting, 0 eligible, 16 blocking 2023-09-29T06:13:18,775 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2023-09-29T06:13:18,775 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2023-09-29T06:13:18,775 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. because compaction request was cancelled 2023-09-29T06:13:18,775 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1291e331f06575ebce0d38fd8ca0d9de:C 2023-09-29T06:13:18,775 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 7de828c345454202b934a75bbb6d2451, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1695967996983 2023-09-29T06:13:18,775 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2023-09-29T06:13:18,776 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting fea8317613ec48d9a42058ef39d1b830, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1695967997748 2023-09-29T06:13:18,776 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 7c09918d63564284a3d8885530d6b18f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1695967997837 2023-09-29T06:13:18,777 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2023-09-29T06:13:18,777 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2023-09-29T06:13:18,777 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting b36fd3ea48e74b798497df97a19b0e90, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1695967997965 2023-09-29T06:13:18,777 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. because compaction request was cancelled 2023-09-29T06:13:18,777 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1291e331f06575ebce0d38fd8ca0d9de:B 2023-09-29T06:13:18,777 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 1bb3b1f26ea84029ba07aad4338133d8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1695967998121 2023-09-29T06:13:18,797 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1291e331f06575ebce0d38fd8ca0d9de#C#compaction#96 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:18,797 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/a58547338ae14bad8be5d5ebfc27ca02 is 50, key is test_row_0/C:col10/1695967998235/Put/seqid=0 2023-09-29T06:13:18,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=30 2023-09-29T06:13:18,918 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=31 2023-09-29T06:13:18,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:18,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. as already flushing 2023-09-29T06:13:18,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:18,919 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=31}] handler.RSProcedureHandler(53): pid=31 java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:18,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=31 java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:18,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=31 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:18,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:18,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968058947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:18,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:18,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:18,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968058948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:18,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968058949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:18,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:18,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968058950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:19,038 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/31cf797c975749b0918f22b71066ee01 2023-09-29T06:13:19,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/b551bdf098654f329e51dd6757ea29d2 is 50, key is test_row_0/C:col10/1695967998263/Put/seqid=0 2023-09-29T06:13:19,055 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/b551bdf098654f329e51dd6757ea29d2 2023-09-29T06:13:19,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/06b6f949408b4e95be3e89d4fb83b0a2 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/06b6f949408b4e95be3e89d4fb83b0a2 2023-09-29T06:13:19,071 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/06b6f949408b4e95be3e89d4fb83b0a2, entries=200, sequenceid=464, filesize=14.4 K 2023-09-29T06:13:19,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/31cf797c975749b0918f22b71066ee01 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/31cf797c975749b0918f22b71066ee01 2023-09-29T06:13:19,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=31 2023-09-29T06:13:19,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:19,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. as already flushing 2023-09-29T06:13:19,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:19,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=31}] handler.RSProcedureHandler(53): pid=31 java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:19,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=31 java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:19,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=31 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:19,080 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/31cf797c975749b0918f22b71066ee01, entries=150, sequenceid=464, filesize=12.0 K 2023-09-29T06:13:19,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/b551bdf098654f329e51dd6757ea29d2 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/b551bdf098654f329e51dd6757ea29d2 2023-09-29T06:13:19,089 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/b551bdf098654f329e51dd6757ea29d2, entries=150, sequenceid=464, filesize=12.0 K 2023-09-29T06:13:19,090 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 1291e331f06575ebce0d38fd8ca0d9de in 514ms, sequenceid=464, compaction requested=true 2023-09-29T06:13:19,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:19,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1291e331f06575ebce0d38fd8ca0d9de:A, priority=-2147483648, current under compaction store size is 1 2023-09-29T06:13:19,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:19,090 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:13:19,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1291e331f06575ebce0d38fd8ca0d9de:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:13:19,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:13:19,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1291e331f06575ebce0d38fd8ca0d9de:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:13:19,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2023-09-29T06:13:19,092 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45194 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:13:19,092 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 1291e331f06575ebce0d38fd8ca0d9de/A is initiating minor compaction (all files) 2023-09-29T06:13:19,092 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1291e331f06575ebce0d38fd8ca0d9de/A in TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:19,092 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/aad8e39a39904d5cafdeeb0f4585d6cd, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/13d4415cb3a2450c901b16d417402092, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/06b6f949408b4e95be3e89d4fb83b0a2] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp, totalSize=44.1 K 2023-09-29T06:13:19,093 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting aad8e39a39904d5cafdeeb0f4585d6cd, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1695967997965 2023-09-29T06:13:19,093 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 13d4415cb3a2450c901b16d417402092, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1695967998121 2023-09-29T06:13:19,093 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 06b6f949408b4e95be3e89d4fb83b0a2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=464, earliestPutTs=1695967998242 2023-09-29T06:13:19,102 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1291e331f06575ebce0d38fd8ca0d9de#A#compaction#98 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:19,103 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/9aebd9b0a4eb47c7bd69bf659114f356 is 50, key is test_row_0/A:col10/1695967998263/Put/seqid=0 2023-09-29T06:13:19,127 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/9aebd9b0a4eb47c7bd69bf659114f356 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/9aebd9b0a4eb47c7bd69bf659114f356 2023-09-29T06:13:19,139 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 1291e331f06575ebce0d38fd8ca0d9de/A of 1291e331f06575ebce0d38fd8ca0d9de into 9aebd9b0a4eb47c7bd69bf659114f356(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:19,139 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:19,139 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de., storeName=1291e331f06575ebce0d38fd8ca0d9de/A, priority=13, startTime=1695967999090; duration=0sec 2023-09-29T06:13:19,139 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2023-09-29T06:13:19,139 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1291e331f06575ebce0d38fd8ca0d9de:A 2023-09-29T06:13:19,139 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 5 compacting, 1 eligible, 16 blocking 2023-09-29T06:13:19,140 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2023-09-29T06:13:19,140 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2023-09-29T06:13:19,140 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. because compaction request was cancelled 2023-09-29T06:13:19,140 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1291e331f06575ebce0d38fd8ca0d9de:C 2023-09-29T06:13:19,140 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:13:19,142 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37874 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:13:19,142 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 1291e331f06575ebce0d38fd8ca0d9de/B is initiating minor compaction (all files) 2023-09-29T06:13:19,142 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1291e331f06575ebce0d38fd8ca0d9de/B in TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:19,142 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/e79c5123683b4c57b9715dae4a062a26, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/f14a987c2aca43c4be9f2f45ad72cecb, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/31cf797c975749b0918f22b71066ee01] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp, totalSize=37.0 K 2023-09-29T06:13:19,143 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting e79c5123683b4c57b9715dae4a062a26, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1695967997965 2023-09-29T06:13:19,143 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting f14a987c2aca43c4be9f2f45ad72cecb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1695967998121 2023-09-29T06:13:19,143 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 31cf797c975749b0918f22b71066ee01, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=464, earliestPutTs=1695967998242 2023-09-29T06:13:19,156 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1291e331f06575ebce0d38fd8ca0d9de#B#compaction#99 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:19,156 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/611974814589440abcbfe7fe80c73d54 is 50, key is test_row_0/B:col10/1695967998263/Put/seqid=0 2023-09-29T06:13:19,216 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/a58547338ae14bad8be5d5ebfc27ca02 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/a58547338ae14bad8be5d5ebfc27ca02 2023-09-29T06:13:19,224 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 5 (all) file(s) in 1291e331f06575ebce0d38fd8ca0d9de/C of 1291e331f06575ebce0d38fd8ca0d9de into a58547338ae14bad8be5d5ebfc27ca02(size=13.0 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:19,224 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:19,224 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de., storeName=1291e331f06575ebce0d38fd8ca0d9de/C, priority=11, startTime=1695967998475; duration=0sec 2023-09-29T06:13:19,224 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:19,224 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1291e331f06575ebce0d38fd8ca0d9de:C 2023-09-29T06:13:19,230 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=31 2023-09-29T06:13:19,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:19,230 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.HRegion(2819): Flushing 1291e331f06575ebce0d38fd8ca0d9de 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2023-09-29T06:13:19,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=A 2023-09-29T06:13:19,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:19,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=B 2023-09-29T06:13:19,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:19,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=C 2023-09-29T06:13:19,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:19,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/f8bd2e7453b042aaa30880903984fcff is 50, key is test_row_0/A:col10/1695967998632/Put/seqid=0 2023-09-29T06:13:19,250 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/f8bd2e7453b042aaa30880903984fcff 2023-09-29T06:13:19,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 1291e331f06575ebce0d38fd8ca0d9de 2023-09-29T06:13:19,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. as already flushing 2023-09-29T06:13:19,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/a8f386a9505e4017a1b3e4e45a275f9e is 50, key is test_row_0/B:col10/1695967998632/Put/seqid=0 2023-09-29T06:13:19,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:19,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968059273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:19,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:19,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968059275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:19,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:19,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968059275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:19,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:19,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968059275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:19,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=30 2023-09-29T06:13:19,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:19,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:19,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968059378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:19,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968059378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:19,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:19,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:19,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968059378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:19,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968059379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:19,584 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/611974814589440abcbfe7fe80c73d54 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/611974814589440abcbfe7fe80c73d54 2023-09-29T06:13:19,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:19,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:19,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968059585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:19,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968059585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:19,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:19,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968059586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:19,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:19,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968059590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:19,592 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 1291e331f06575ebce0d38fd8ca0d9de/B of 1291e331f06575ebce0d38fd8ca0d9de into 611974814589440abcbfe7fe80c73d54(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:19,592 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:19,592 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de., storeName=1291e331f06575ebce0d38fd8ca0d9de/B, priority=13, startTime=1695967999090; duration=0sec 2023-09-29T06:13:19,592 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:19,592 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1291e331f06575ebce0d38fd8ca0d9de:B 2023-09-29T06:13:19,687 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/a8f386a9505e4017a1b3e4e45a275f9e 2023-09-29T06:13:19,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/9393dbc4a0044b72b0de077758cdbbfb is 50, key is test_row_0/C:col10/1695967998632/Put/seqid=0 2023-09-29T06:13:19,714 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/9393dbc4a0044b72b0de077758cdbbfb 2023-09-29T06:13:19,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/f8bd2e7453b042aaa30880903984fcff as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/f8bd2e7453b042aaa30880903984fcff 2023-09-29T06:13:19,729 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/f8bd2e7453b042aaa30880903984fcff, entries=150, sequenceid=493, filesize=12.0 K 2023-09-29T06:13:19,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/a8f386a9505e4017a1b3e4e45a275f9e as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/a8f386a9505e4017a1b3e4e45a275f9e 2023-09-29T06:13:19,739 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/a8f386a9505e4017a1b3e4e45a275f9e, entries=150, sequenceid=493, filesize=12.0 K 2023-09-29T06:13:19,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/9393dbc4a0044b72b0de077758cdbbfb as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/9393dbc4a0044b72b0de077758cdbbfb 2023-09-29T06:13:19,750 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/9393dbc4a0044b72b0de077758cdbbfb, entries=150, sequenceid=493, filesize=12.0 K 2023-09-29T06:13:19,751 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.HRegion(3022): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 1291e331f06575ebce0d38fd8ca0d9de in 521ms, sequenceid=493, compaction requested=true 2023-09-29T06:13:19,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.HRegion(2520): Flush status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:19,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:19,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=31 2023-09-29T06:13:19,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=31 2023-09-29T06:13:19,759 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=31, resume processing ppid=30 2023-09-29T06:13:19,759 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=31, ppid=30, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4920 sec 2023-09-29T06:13:19,761 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=30, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=30, table=TestAcidGuarantees in 1.5120 sec 2023-09-29T06:13:19,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 1291e331f06575ebce0d38fd8ca0d9de 2023-09-29T06:13:19,890 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 1291e331f06575ebce0d38fd8ca0d9de 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2023-09-29T06:13:19,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=A 2023-09-29T06:13:19,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:19,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=B 2023-09-29T06:13:19,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:19,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=C 2023-09-29T06:13:19,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:19,897 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/ce52e15280e34c62a2b277ede1a888d9 is 50, key is test_row_0/A:col10/1695967999272/Put/seqid=0 2023-09-29T06:13:19,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:19,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968059925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:19,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:19,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968059925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:19,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:19,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968059928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:19,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:19,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968059931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:20,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:20,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968060031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:20,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:20,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968060033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:20,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:20,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968060036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:20,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:20,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968060039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:20,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:20,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968060233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:20,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:20,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968060236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:20,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:20,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968060244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:20,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:20,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968060244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:20,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:20,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54718 deadline: 1695968060260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:20,271 DEBUG [Thread-142 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=7, retries=16, started=8190 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_0' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de., hostname=jenkins-hbase3.apache.org,39783,1695967978138, seqNum=2, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:572) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:559) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:197) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:218) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:750) 2023-09-29T06:13:20,319 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=507 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/ce52e15280e34c62a2b277ede1a888d9 2023-09-29T06:13:20,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/637580ed3836447ebb713325f608c691 is 50, key is test_row_0/B:col10/1695967999272/Put/seqid=0 2023-09-29T06:13:20,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=30 2023-09-29T06:13:20,356 INFO [Thread-152 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 30 completed 2023-09-29T06:13:20,359 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:13:20,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=32, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=32, table=TestAcidGuarantees 2023-09-29T06:13:20,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-09-29T06:13:20,368 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=32, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=32, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:13:20,371 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=32, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=32, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:13:20,371 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:13:20,391 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=507 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/637580ed3836447ebb713325f608c691 2023-09-29T06:13:20,400 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/88906a25f53c4a6f83dc4f571d4129bc is 50, key is test_row_0/C:col10/1695967999272/Put/seqid=0 2023-09-29T06:13:20,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-09-29T06:13:20,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2023-09-29T06:13:20,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:20,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:20,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. as already flushing 2023-09-29T06:13:20,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968060538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:20,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:20,539 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(53): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:20,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:20,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:20,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968060539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:20,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:20,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:20,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968060548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:20,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:20,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968060549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:20,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-09-29T06:13:20,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2023-09-29T06:13:20,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:20,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. as already flushing 2023-09-29T06:13:20,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:20,693 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(53): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:20,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:20,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:20,816 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=507 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/88906a25f53c4a6f83dc4f571d4129bc 2023-09-29T06:13:20,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/ce52e15280e34c62a2b277ede1a888d9 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/ce52e15280e34c62a2b277ede1a888d9 2023-09-29T06:13:20,841 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/ce52e15280e34c62a2b277ede1a888d9, entries=150, sequenceid=507, filesize=12.0 K 2023-09-29T06:13:20,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/637580ed3836447ebb713325f608c691 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/637580ed3836447ebb713325f608c691 2023-09-29T06:13:20,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/637580ed3836447ebb713325f608c691, entries=150, sequenceid=507, filesize=12.0 K 2023-09-29T06:13:20,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/88906a25f53c4a6f83dc4f571d4129bc as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/88906a25f53c4a6f83dc4f571d4129bc 2023-09-29T06:13:20,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2023-09-29T06:13:20,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:20,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. as already flushing 2023-09-29T06:13:20,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:20,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(53): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:20,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:20,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:20,859 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/88906a25f53c4a6f83dc4f571d4129bc, entries=150, sequenceid=507, filesize=12.0 K 2023-09-29T06:13:20,860 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 1291e331f06575ebce0d38fd8ca0d9de in 970ms, sequenceid=507, compaction requested=true 2023-09-29T06:13:20,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:20,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1291e331f06575ebce0d38fd8ca0d9de:A, priority=-2147483648, current under compaction store size is 1 2023-09-29T06:13:20,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:20,860 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:13:20,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1291e331f06575ebce0d38fd8ca0d9de:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:13:20,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:20,861 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:13:20,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1291e331f06575ebce0d38fd8ca0d9de:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:13:20,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:13:20,862 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37976 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:13:20,862 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 1291e331f06575ebce0d38fd8ca0d9de/A is initiating minor compaction (all files) 2023-09-29T06:13:20,862 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1291e331f06575ebce0d38fd8ca0d9de/A in TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:20,862 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/9aebd9b0a4eb47c7bd69bf659114f356, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/f8bd2e7453b042aaa30880903984fcff, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/ce52e15280e34c62a2b277ede1a888d9] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp, totalSize=37.1 K 2023-09-29T06:13:20,863 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37976 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:13:20,863 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 1291e331f06575ebce0d38fd8ca0d9de/B is initiating minor compaction (all files) 2023-09-29T06:13:20,863 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1291e331f06575ebce0d38fd8ca0d9de/B in TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:20,863 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/611974814589440abcbfe7fe80c73d54, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/a8f386a9505e4017a1b3e4e45a275f9e, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/637580ed3836447ebb713325f608c691] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp, totalSize=37.1 K 2023-09-29T06:13:20,863 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 9aebd9b0a4eb47c7bd69bf659114f356, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=464, earliestPutTs=1695967998242 2023-09-29T06:13:20,863 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 611974814589440abcbfe7fe80c73d54, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=464, earliestPutTs=1695967998242 2023-09-29T06:13:20,864 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting f8bd2e7453b042aaa30880903984fcff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1695967998617 2023-09-29T06:13:20,865 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting a8f386a9505e4017a1b3e4e45a275f9e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1695967998617 2023-09-29T06:13:20,865 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting ce52e15280e34c62a2b277ede1a888d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=507, earliestPutTs=1695967999263 2023-09-29T06:13:20,867 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 637580ed3836447ebb713325f608c691, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=507, earliestPutTs=1695967999263 2023-09-29T06:13:20,880 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1291e331f06575ebce0d38fd8ca0d9de#A#compaction#106 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:20,880 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/1706e27398ae44acb2d4c4cf7e1a1e2c is 50, key is test_row_0/A:col10/1695967999272/Put/seqid=0 2023-09-29T06:13:20,885 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1291e331f06575ebce0d38fd8ca0d9de#B#compaction#107 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:20,885 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/4d1c6ef96c0b488ca27ad9ad5defe599 is 50, key is test_row_0/B:col10/1695967999272/Put/seqid=0 2023-09-29T06:13:20,903 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/1706e27398ae44acb2d4c4cf7e1a1e2c as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/1706e27398ae44acb2d4c4cf7e1a1e2c 2023-09-29T06:13:20,904 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/4d1c6ef96c0b488ca27ad9ad5defe599 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/4d1c6ef96c0b488ca27ad9ad5defe599 2023-09-29T06:13:20,909 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 1291e331f06575ebce0d38fd8ca0d9de/A of 1291e331f06575ebce0d38fd8ca0d9de into 1706e27398ae44acb2d4c4cf7e1a1e2c(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:20,909 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:20,910 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de., storeName=1291e331f06575ebce0d38fd8ca0d9de/A, priority=13, startTime=1695968000860; duration=0sec 2023-09-29T06:13:20,910 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:13:20,910 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1291e331f06575ebce0d38fd8ca0d9de:A 2023-09-29T06:13:20,910 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-29T06:13:20,912 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 1291e331f06575ebce0d38fd8ca0d9de/B of 1291e331f06575ebce0d38fd8ca0d9de into 4d1c6ef96c0b488ca27ad9ad5defe599(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:20,912 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:20,912 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de., storeName=1291e331f06575ebce0d38fd8ca0d9de/B, priority=13, startTime=1695968000860; duration=0sec 2023-09-29T06:13:20,912 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:20,912 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1291e331f06575ebce0d38fd8ca0d9de:B 2023-09-29T06:13:20,912 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50158 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-29T06:13:20,912 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 1291e331f06575ebce0d38fd8ca0d9de/C is initiating minor compaction (all files) 2023-09-29T06:13:20,913 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1291e331f06575ebce0d38fd8ca0d9de/C in TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:20,913 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/a58547338ae14bad8be5d5ebfc27ca02, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/b551bdf098654f329e51dd6757ea29d2, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/9393dbc4a0044b72b0de077758cdbbfb, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/88906a25f53c4a6f83dc4f571d4129bc] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp, totalSize=49.0 K 2023-09-29T06:13:20,913 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting a58547338ae14bad8be5d5ebfc27ca02, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1695967998121 2023-09-29T06:13:20,914 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting b551bdf098654f329e51dd6757ea29d2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=464, earliestPutTs=1695967998242 2023-09-29T06:13:20,914 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 9393dbc4a0044b72b0de077758cdbbfb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1695967998617 2023-09-29T06:13:20,915 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 88906a25f53c4a6f83dc4f571d4129bc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=507, earliestPutTs=1695967999263 2023-09-29T06:13:20,931 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1291e331f06575ebce0d38fd8ca0d9de#C#compaction#108 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:20,931 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/0dfd10fc97d145dd9f3d0a25220e7fc5 is 50, key is test_row_0/C:col10/1695967999272/Put/seqid=0 2023-09-29T06:13:20,944 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/0dfd10fc97d145dd9f3d0a25220e7fc5 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/0dfd10fc97d145dd9f3d0a25220e7fc5 2023-09-29T06:13:20,952 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 1291e331f06575ebce0d38fd8ca0d9de/C of 1291e331f06575ebce0d38fd8ca0d9de into 0dfd10fc97d145dd9f3d0a25220e7fc5(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:20,952 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:20,952 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de., storeName=1291e331f06575ebce0d38fd8ca0d9de/C, priority=12, startTime=1695968000861; duration=0sec 2023-09-29T06:13:20,952 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:20,952 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1291e331f06575ebce0d38fd8ca0d9de:C 2023-09-29T06:13:20,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-09-29T06:13:21,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2023-09-29T06:13:21,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:21,008 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2819): Flushing 1291e331f06575ebce0d38fd8ca0d9de 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2023-09-29T06:13:21,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=A 2023-09-29T06:13:21,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:21,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=B 2023-09-29T06:13:21,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:21,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=C 2023-09-29T06:13:21,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:21,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/d4fe7d3da5db4623b1a54af454ebfc89 is 50, key is test_row_0/A:col10/1695967999927/Put/seqid=0 2023-09-29T06:13:21,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 1291e331f06575ebce0d38fd8ca0d9de 2023-09-29T06:13:21,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. as already flushing 2023-09-29T06:13:21,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:21,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968061052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:21,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:21,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968061054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:21,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:21,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968061056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:21,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:21,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968061057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:21,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:21,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968061158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:21,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:21,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968061158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:21,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:21,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968061161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:21,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:21,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968061360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:21,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:21,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968061361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:21,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:21,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968061363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:21,427 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/d4fe7d3da5db4623b1a54af454ebfc89 2023-09-29T06:13:21,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/889b98d927c6432cb69cb1db95d1c01e is 50, key is test_row_0/B:col10/1695967999927/Put/seqid=0 2023-09-29T06:13:21,447 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/889b98d927c6432cb69cb1db95d1c01e 2023-09-29T06:13:21,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-09-29T06:13:21,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/bf455c93a87f4d31bf6f9971c06b3c29 is 50, key is test_row_0/C:col10/1695967999927/Put/seqid=0 2023-09-29T06:13:21,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:21,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968061666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:21,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:21,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968061667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:21,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:21,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968061667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:21,894 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/bf455c93a87f4d31bf6f9971c06b3c29 2023-09-29T06:13:21,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/d4fe7d3da5db4623b1a54af454ebfc89 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/d4fe7d3da5db4623b1a54af454ebfc89 2023-09-29T06:13:21,907 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/d4fe7d3da5db4623b1a54af454ebfc89, entries=150, sequenceid=534, filesize=12.0 K 2023-09-29T06:13:21,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/889b98d927c6432cb69cb1db95d1c01e as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/889b98d927c6432cb69cb1db95d1c01e 2023-09-29T06:13:21,918 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/889b98d927c6432cb69cb1db95d1c01e, entries=150, sequenceid=534, filesize=12.0 K 2023-09-29T06:13:21,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/bf455c93a87f4d31bf6f9971c06b3c29 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/bf455c93a87f4d31bf6f9971c06b3c29 2023-09-29T06:13:21,925 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/bf455c93a87f4d31bf6f9971c06b3c29, entries=150, sequenceid=534, filesize=12.0 K 2023-09-29T06:13:21,926 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(3022): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 1291e331f06575ebce0d38fd8ca0d9de in 918ms, sequenceid=534, compaction requested=false 2023-09-29T06:13:21,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2520): Flush status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:21,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:21,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=33 2023-09-29T06:13:21,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=33 2023-09-29T06:13:21,933 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=33, resume processing ppid=32 2023-09-29T06:13:21,933 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=33, ppid=32, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5600 sec 2023-09-29T06:13:21,935 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=32, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=32, table=TestAcidGuarantees in 1.5740 sec 2023-09-29T06:13:22,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 1291e331f06575ebce0d38fd8ca0d9de 2023-09-29T06:13:22,065 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 1291e331f06575ebce0d38fd8ca0d9de 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2023-09-29T06:13:22,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=A 2023-09-29T06:13:22,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:22,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=B 2023-09-29T06:13:22,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:22,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=C 2023-09-29T06:13:22,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:22,070 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/80265bc85904496d9ff18587eb780290 is 50, key is test_row_0/A:col10/1695968001052/Put/seqid=0 2023-09-29T06:13:22,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:22,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 251 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968062113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:22,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:22,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968062170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:22,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:22,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968062174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:22,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:22,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968062174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:22,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:22,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968062217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:22,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:22,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968062418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:22,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-09-29T06:13:22,467 INFO [Thread-152 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 32 completed 2023-09-29T06:13:22,469 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:13:22,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=34, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees 2023-09-29T06:13:22,472 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=34, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:13:22,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-09-29T06:13:22,473 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=34, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:13:22,473 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:13:22,481 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=547 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/80265bc85904496d9ff18587eb780290 2023-09-29T06:13:22,492 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/469e52dce4d74160989cd6b68a230c09 is 50, key is test_row_0/B:col10/1695968001052/Put/seqid=0 2023-09-29T06:13:22,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-09-29T06:13:22,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2023-09-29T06:13:22,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:22,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. as already flushing 2023-09-29T06:13:22,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:22,631 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(53): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:22,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:22,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:22,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:22,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968062723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:22,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-09-29T06:13:22,788 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2023-09-29T06:13:22,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:22,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. as already flushing 2023-09-29T06:13:22,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:22,789 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(53): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:22,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:22,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:22,921 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=547 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/469e52dce4d74160989cd6b68a230c09 2023-09-29T06:13:22,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2023-09-29T06:13:22,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:22,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. as already flushing 2023-09-29T06:13:22,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:22,944 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(53): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:22,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:22,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:22,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/dc99aa22cbcc4c938ff14991de63f517 is 50, key is test_row_0/C:col10/1695968001052/Put/seqid=0 2023-09-29T06:13:22,967 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=547 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/dc99aa22cbcc4c938ff14991de63f517 2023-09-29T06:13:22,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/80265bc85904496d9ff18587eb780290 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/80265bc85904496d9ff18587eb780290 2023-09-29T06:13:22,980 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/80265bc85904496d9ff18587eb780290, entries=200, sequenceid=547, filesize=14.4 K 2023-09-29T06:13:22,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/469e52dce4d74160989cd6b68a230c09 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/469e52dce4d74160989cd6b68a230c09 2023-09-29T06:13:22,990 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/469e52dce4d74160989cd6b68a230c09, entries=150, sequenceid=547, filesize=12.0 K 2023-09-29T06:13:22,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/dc99aa22cbcc4c938ff14991de63f517 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/dc99aa22cbcc4c938ff14991de63f517 2023-09-29T06:13:23,009 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/dc99aa22cbcc4c938ff14991de63f517, entries=150, sequenceid=547, filesize=12.0 K 2023-09-29T06:13:23,010 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 1291e331f06575ebce0d38fd8ca0d9de in 944ms, sequenceid=547, compaction requested=true 2023-09-29T06:13:23,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:23,010 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:13:23,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1291e331f06575ebce0d38fd8ca0d9de:A, priority=-2147483648, current under compaction store size is 1 2023-09-29T06:13:23,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:23,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1291e331f06575ebce0d38fd8ca0d9de:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:13:23,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:23,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1291e331f06575ebce0d38fd8ca0d9de:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:13:23,010 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:13:23,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:13:23,012 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40518 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:13:23,012 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 1291e331f06575ebce0d38fd8ca0d9de/A is initiating minor compaction (all files) 2023-09-29T06:13:23,012 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38078 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:13:23,012 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1291e331f06575ebce0d38fd8ca0d9de/A in TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:23,012 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 1291e331f06575ebce0d38fd8ca0d9de/B is initiating minor compaction (all files) 2023-09-29T06:13:23,012 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/1706e27398ae44acb2d4c4cf7e1a1e2c, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/d4fe7d3da5db4623b1a54af454ebfc89, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/80265bc85904496d9ff18587eb780290] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp, totalSize=39.6 K 2023-09-29T06:13:23,012 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1291e331f06575ebce0d38fd8ca0d9de/B in TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:23,012 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/4d1c6ef96c0b488ca27ad9ad5defe599, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/889b98d927c6432cb69cb1db95d1c01e, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/469e52dce4d74160989cd6b68a230c09] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp, totalSize=37.2 K 2023-09-29T06:13:23,013 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 1706e27398ae44acb2d4c4cf7e1a1e2c, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=507, earliestPutTs=1695967999263 2023-09-29T06:13:23,013 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 4d1c6ef96c0b488ca27ad9ad5defe599, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=507, earliestPutTs=1695967999263 2023-09-29T06:13:23,013 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting d4fe7d3da5db4623b1a54af454ebfc89, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1695967999924 2023-09-29T06:13:23,013 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 889b98d927c6432cb69cb1db95d1c01e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1695967999924 2023-09-29T06:13:23,014 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 80265bc85904496d9ff18587eb780290, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=547, earliestPutTs=1695968001051 2023-09-29T06:13:23,014 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 469e52dce4d74160989cd6b68a230c09, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=547, earliestPutTs=1695968001051 2023-09-29T06:13:23,023 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1291e331f06575ebce0d38fd8ca0d9de#B#compaction#115 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:23,023 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1291e331f06575ebce0d38fd8ca0d9de#A#compaction#116 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:23,023 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/ac14ae6b9c7945cb911e3daf8e882fbd is 50, key is test_row_0/B:col10/1695968001052/Put/seqid=0 2023-09-29T06:13:23,024 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/ede78ea2cfc049bf89716fad95f1929a is 50, key is test_row_0/A:col10/1695968001052/Put/seqid=0 2023-09-29T06:13:23,052 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/ede78ea2cfc049bf89716fad95f1929a as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/ede78ea2cfc049bf89716fad95f1929a 2023-09-29T06:13:23,054 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/ac14ae6b9c7945cb911e3daf8e882fbd as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/ac14ae6b9c7945cb911e3daf8e882fbd 2023-09-29T06:13:23,059 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 1291e331f06575ebce0d38fd8ca0d9de/A of 1291e331f06575ebce0d38fd8ca0d9de into ede78ea2cfc049bf89716fad95f1929a(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:23,059 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:23,060 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de., storeName=1291e331f06575ebce0d38fd8ca0d9de/A, priority=13, startTime=1695968003010; duration=0sec 2023-09-29T06:13:23,060 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:13:23,060 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1291e331f06575ebce0d38fd8ca0d9de:A 2023-09-29T06:13:23,060 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:13:23,062 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38010 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:13:23,062 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 1291e331f06575ebce0d38fd8ca0d9de/C is initiating minor compaction (all files) 2023-09-29T06:13:23,062 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1291e331f06575ebce0d38fd8ca0d9de/C in TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:23,063 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/0dfd10fc97d145dd9f3d0a25220e7fc5, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/bf455c93a87f4d31bf6f9971c06b3c29, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/dc99aa22cbcc4c938ff14991de63f517] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp, totalSize=37.1 K 2023-09-29T06:13:23,066 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 0dfd10fc97d145dd9f3d0a25220e7fc5, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=507, earliestPutTs=1695967999263 2023-09-29T06:13:23,067 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting bf455c93a87f4d31bf6f9971c06b3c29, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1695967999924 2023-09-29T06:13:23,067 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting dc99aa22cbcc4c938ff14991de63f517, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=547, earliestPutTs=1695968001051 2023-09-29T06:13:23,068 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 1291e331f06575ebce0d38fd8ca0d9de/B of 1291e331f06575ebce0d38fd8ca0d9de into ac14ae6b9c7945cb911e3daf8e882fbd(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:23,068 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:23,068 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de., storeName=1291e331f06575ebce0d38fd8ca0d9de/B, priority=13, startTime=1695968003010; duration=0sec 2023-09-29T06:13:23,068 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:23,068 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1291e331f06575ebce0d38fd8ca0d9de:B 2023-09-29T06:13:23,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-09-29T06:13:23,077 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1291e331f06575ebce0d38fd8ca0d9de#C#compaction#117 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:23,078 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/2e230bcc8fb14029baa49a9ffbe7015d is 50, key is test_row_0/C:col10/1695968001052/Put/seqid=0 2023-09-29T06:13:23,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2023-09-29T06:13:23,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:23,098 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2819): Flushing 1291e331f06575ebce0d38fd8ca0d9de 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2023-09-29T06:13:23,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=A 2023-09-29T06:13:23,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:23,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=B 2023-09-29T06:13:23,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:23,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=C 2023-09-29T06:13:23,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:23,102 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/2e230bcc8fb14029baa49a9ffbe7015d as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/2e230bcc8fb14029baa49a9ffbe7015d 2023-09-29T06:13:23,111 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 1291e331f06575ebce0d38fd8ca0d9de/C of 1291e331f06575ebce0d38fd8ca0d9de into 2e230bcc8fb14029baa49a9ffbe7015d(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:23,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/a148d1d401d04fddbd131e4af96cc14a is 50, key is test_row_0/A:col10/1695968002107/Put/seqid=0 2023-09-29T06:13:23,111 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:23,111 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de., storeName=1291e331f06575ebce0d38fd8ca0d9de/C, priority=13, startTime=1695968003010; duration=0sec 2023-09-29T06:13:23,111 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:23,111 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1291e331f06575ebce0d38fd8ca0d9de:C 2023-09-29T06:13:23,122 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=572 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/a148d1d401d04fddbd131e4af96cc14a 2023-09-29T06:13:23,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/506cbd5cd00040bf864e31232f391701 is 50, key is test_row_0/B:col10/1695968002107/Put/seqid=0 2023-09-29T06:13:23,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 1291e331f06575ebce0d38fd8ca0d9de 2023-09-29T06:13:23,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. as already flushing 2023-09-29T06:13:23,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:23,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968063193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:23,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:23,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968063193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:23,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:23,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 250 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968063194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:23,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:23,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 259 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54756 deadline: 1695968063225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:23,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:23,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:23,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968063297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:23,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968063297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:23,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:23,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968063298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:23,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:23,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968063500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:23,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:23,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968063500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:23,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:23,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968063501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:23,554 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=572 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/506cbd5cd00040bf864e31232f391701 2023-09-29T06:13:23,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/bb17ec84de814123b3b9aaa4326b6aee is 50, key is test_row_0/C:col10/1695968002107/Put/seqid=0 2023-09-29T06:13:23,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-09-29T06:13:23,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:23,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54780 deadline: 1695968063804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:23,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:23,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54730 deadline: 1695968063804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:23,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:23,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:54760 deadline: 1695968063815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:23,972 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=572 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/bb17ec84de814123b3b9aaa4326b6aee 2023-09-29T06:13:23,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/a148d1d401d04fddbd131e4af96cc14a as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/a148d1d401d04fddbd131e4af96cc14a 2023-09-29T06:13:23,988 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/a148d1d401d04fddbd131e4af96cc14a, entries=150, sequenceid=572, filesize=12.0 K 2023-09-29T06:13:23,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/506cbd5cd00040bf864e31232f391701 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/506cbd5cd00040bf864e31232f391701 2023-09-29T06:13:23,995 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/506cbd5cd00040bf864e31232f391701, entries=150, sequenceid=572, filesize=12.0 K 2023-09-29T06:13:23,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/bb17ec84de814123b3b9aaa4326b6aee as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/bb17ec84de814123b3b9aaa4326b6aee 2023-09-29T06:13:24,001 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/bb17ec84de814123b3b9aaa4326b6aee, entries=150, sequenceid=572, filesize=12.0 K 2023-09-29T06:13:24,002 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(3022): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 1291e331f06575ebce0d38fd8ca0d9de in 904ms, sequenceid=572, compaction requested=false 2023-09-29T06:13:24,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2520): Flush status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:24,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:24,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=35 2023-09-29T06:13:24,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=35 2023-09-29T06:13:24,006 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=35, resume processing ppid=34 2023-09-29T06:13:24,006 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=35, ppid=34, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5310 sec 2023-09-29T06:13:24,008 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=34, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees in 1.5380 sec 2023-09-29T06:13:24,066 DEBUG [Thread-155 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x3820db30 to 127.0.0.1:49344 2023-09-29T06:13:24,066 DEBUG [Thread-157 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x3dfed535 to 127.0.0.1:49344 2023-09-29T06:13:24,066 DEBUG [Thread-155 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:13:24,066 DEBUG [Thread-157 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:13:24,067 DEBUG [Thread-153 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x171d09b4 to 127.0.0.1:49344 2023-09-29T06:13:24,067 DEBUG [Thread-153 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:13:24,073 DEBUG [Thread-159 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0035664b to 127.0.0.1:49344 2023-09-29T06:13:24,073 DEBUG [Thread-159 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:13:24,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 1291e331f06575ebce0d38fd8ca0d9de 2023-09-29T06:13:24,233 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 1291e331f06575ebce0d38fd8ca0d9de 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2023-09-29T06:13:24,233 DEBUG [Thread-150 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x3bf36245 to 127.0.0.1:49344 2023-09-29T06:13:24,233 DEBUG [Thread-150 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:13:24,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=A 2023-09-29T06:13:24,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:24,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=B 2023-09-29T06:13:24,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:24,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=C 2023-09-29T06:13:24,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:24,238 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/e90bbd9f5f974e518d8c16ea5266a85b is 50, key is test_row_0/A:col10/1695968004232/Put/seqid=0 2023-09-29T06:13:24,243 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=588 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/e90bbd9f5f974e518d8c16ea5266a85b 2023-09-29T06:13:24,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/0ad4b0b197c0440cb9e25584001a3d7b is 50, key is test_row_0/B:col10/1695968004232/Put/seqid=0 2023-09-29T06:13:24,258 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=588 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/0ad4b0b197c0440cb9e25584001a3d7b 2023-09-29T06:13:24,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/f1424248f43940e69f7cce25d5d37c25 is 50, key is test_row_0/C:col10/1695968004232/Put/seqid=0 2023-09-29T06:13:24,273 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=588 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/f1424248f43940e69f7cce25d5d37c25 2023-09-29T06:13:24,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/e90bbd9f5f974e518d8c16ea5266a85b as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/e90bbd9f5f974e518d8c16ea5266a85b 2023-09-29T06:13:24,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/e90bbd9f5f974e518d8c16ea5266a85b, entries=150, sequenceid=588, filesize=12.0 K 2023-09-29T06:13:24,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/0ad4b0b197c0440cb9e25584001a3d7b as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/0ad4b0b197c0440cb9e25584001a3d7b 2023-09-29T06:13:24,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/0ad4b0b197c0440cb9e25584001a3d7b, entries=150, sequenceid=588, filesize=12.0 K 2023-09-29T06:13:24,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/f1424248f43940e69f7cce25d5d37c25 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/f1424248f43940e69f7cce25d5d37c25 2023-09-29T06:13:24,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/f1424248f43940e69f7cce25d5d37c25, entries=150, sequenceid=588, filesize=12.0 K 2023-09-29T06:13:24,298 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=0 B/0 for 1291e331f06575ebce0d38fd8ca0d9de in 65ms, sequenceid=588, compaction requested=true 2023-09-29T06:13:24,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:24,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1291e331f06575ebce0d38fd8ca0d9de:A, priority=-2147483648, current under compaction store size is 1 2023-09-29T06:13:24,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:24,299 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:13:24,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1291e331f06575ebce0d38fd8ca0d9de:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:13:24,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:24,299 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:13:24,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1291e331f06575ebce0d38fd8ca0d9de:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:13:24,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:13:24,300 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38180 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:13:24,300 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 1291e331f06575ebce0d38fd8ca0d9de/A is initiating minor compaction (all files) 2023-09-29T06:13:24,300 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1291e331f06575ebce0d38fd8ca0d9de/A in TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:24,301 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/ede78ea2cfc049bf89716fad95f1929a, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/a148d1d401d04fddbd131e4af96cc14a, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/e90bbd9f5f974e518d8c16ea5266a85b] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp, totalSize=37.3 K 2023-09-29T06:13:24,301 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38180 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:13:24,301 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 1291e331f06575ebce0d38fd8ca0d9de/B is initiating minor compaction (all files) 2023-09-29T06:13:24,301 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1291e331f06575ebce0d38fd8ca0d9de/B in TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:24,301 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting ede78ea2cfc049bf89716fad95f1929a, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=547, earliestPutTs=1695968001051 2023-09-29T06:13:24,301 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/ac14ae6b9c7945cb911e3daf8e882fbd, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/506cbd5cd00040bf864e31232f391701, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/0ad4b0b197c0440cb9e25584001a3d7b] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp, totalSize=37.3 K 2023-09-29T06:13:24,301 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting a148d1d401d04fddbd131e4af96cc14a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=572, earliestPutTs=1695968002100 2023-09-29T06:13:24,301 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting ac14ae6b9c7945cb911e3daf8e882fbd, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=547, earliestPutTs=1695968001051 2023-09-29T06:13:24,302 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting e90bbd9f5f974e518d8c16ea5266a85b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=588, earliestPutTs=1695968003189 2023-09-29T06:13:24,302 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 506cbd5cd00040bf864e31232f391701, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=572, earliestPutTs=1695968002100 2023-09-29T06:13:24,302 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 0ad4b0b197c0440cb9e25584001a3d7b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=588, earliestPutTs=1695968003189 2023-09-29T06:13:24,310 DEBUG [Thread-148 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x70544c9c to 127.0.0.1:49344 2023-09-29T06:13:24,310 DEBUG [Thread-148 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:13:24,310 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1291e331f06575ebce0d38fd8ca0d9de#A#compaction#124 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:24,311 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1291e331f06575ebce0d38fd8ca0d9de#B#compaction#125 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:24,311 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/bfbd24a11b3446ecbd3245bc9057f743 is 50, key is test_row_0/A:col10/1695968004232/Put/seqid=0 2023-09-29T06:13:24,311 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/61013a0b109e4a39a82a964033b7f23e is 50, key is test_row_0/B:col10/1695968004232/Put/seqid=0 2023-09-29T06:13:24,312 DEBUG [Thread-144 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x04ea4d93 to 127.0.0.1:49344 2023-09-29T06:13:24,312 DEBUG [Thread-144 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:13:24,320 DEBUG [Thread-146 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x24625a3b to 127.0.0.1:49344 2023-09-29T06:13:24,320 DEBUG [Thread-146 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:13:24,327 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/bfbd24a11b3446ecbd3245bc9057f743 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/bfbd24a11b3446ecbd3245bc9057f743 2023-09-29T06:13:24,328 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/61013a0b109e4a39a82a964033b7f23e as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/61013a0b109e4a39a82a964033b7f23e 2023-09-29T06:13:24,333 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 1291e331f06575ebce0d38fd8ca0d9de/A of 1291e331f06575ebce0d38fd8ca0d9de into bfbd24a11b3446ecbd3245bc9057f743(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:24,333 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:24,333 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de., storeName=1291e331f06575ebce0d38fd8ca0d9de/A, priority=13, startTime=1695968004299; duration=0sec 2023-09-29T06:13:24,333 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:13:24,333 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1291e331f06575ebce0d38fd8ca0d9de:A 2023-09-29T06:13:24,333 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:13:24,335 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 1291e331f06575ebce0d38fd8ca0d9de/B of 1291e331f06575ebce0d38fd8ca0d9de into 61013a0b109e4a39a82a964033b7f23e(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:24,335 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:24,335 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de., storeName=1291e331f06575ebce0d38fd8ca0d9de/B, priority=13, startTime=1695968004299; duration=0sec 2023-09-29T06:13:24,335 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:24,335 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1291e331f06575ebce0d38fd8ca0d9de:B 2023-09-29T06:13:24,335 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38112 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:13:24,335 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 1291e331f06575ebce0d38fd8ca0d9de/C is initiating minor compaction (all files) 2023-09-29T06:13:24,335 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1291e331f06575ebce0d38fd8ca0d9de/C in TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:24,335 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/2e230bcc8fb14029baa49a9ffbe7015d, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/bb17ec84de814123b3b9aaa4326b6aee, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/f1424248f43940e69f7cce25d5d37c25] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp, totalSize=37.2 K 2023-09-29T06:13:24,338 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 2e230bcc8fb14029baa49a9ffbe7015d, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=547, earliestPutTs=1695968001051 2023-09-29T06:13:24,338 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting bb17ec84de814123b3b9aaa4326b6aee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=572, earliestPutTs=1695968002100 2023-09-29T06:13:24,339 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting f1424248f43940e69f7cce25d5d37c25, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=588, earliestPutTs=1695968003189 2023-09-29T06:13:24,347 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1291e331f06575ebce0d38fd8ca0d9de#C#compaction#126 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:24,347 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/237cc160f985423c96d608feef08e614 is 50, key is test_row_0/C:col10/1695968004232/Put/seqid=0 2023-09-29T06:13:24,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-09-29T06:13:24,577 INFO [Thread-152 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 34 completed 2023-09-29T06:13:24,759 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/237cc160f985423c96d608feef08e614 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/237cc160f985423c96d608feef08e614 2023-09-29T06:13:24,764 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 1291e331f06575ebce0d38fd8ca0d9de/C of 1291e331f06575ebce0d38fd8ca0d9de into 237cc160f985423c96d608feef08e614(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:24,764 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:24,764 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de., storeName=1291e331f06575ebce0d38fd8ca0d9de/C, priority=13, startTime=1695968004299; duration=0sec 2023-09-29T06:13:24,765 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:24,765 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1291e331f06575ebce0d38fd8ca0d9de:C 2023-09-29T06:13:30,295 DEBUG [Thread-142 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x2ee95d5f to 127.0.0.1:49344 2023-09-29T06:13:30,295 DEBUG [Thread-142 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:13:30,295 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2023-09-29T06:13:30,295 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(394): wrote 36 2023-09-29T06:13:30,295 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(394): wrote 105 2023-09-29T06:13:30,295 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(394): wrote 112 2023-09-29T06:13:30,295 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(394): wrote 93 2023-09-29T06:13:30,295 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(394): wrote 121 2023-09-29T06:13:30,296 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(396): Readers: 2023-09-29T06:13:30,296 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(398): read 4880 2023-09-29T06:13:30,296 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(398): read 4707 2023-09-29T06:13:30,296 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2023-09-29T06:13:30,296 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(402): scanned 2259 2023-09-29T06:13:30,296 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(403): verified 6775 rows 2023-09-29T06:13:30,296 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(402): scanned 2237 2023-09-29T06:13:30,296 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(403): verified 6709 rows 2023-09-29T06:13:30,296 INFO [Listener at localhost/42843 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-09-29T06:13:30,296 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x19f53530 to 127.0.0.1:49344 2023-09-29T06:13:30,296 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:13:30,304 INFO [Listener at localhost/42843 {}] client.HBaseAdmin$18(967): Started disable of TestAcidGuarantees 2023-09-29T06:13:30,310 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$12(2664): Client=jenkins//172.31.12.81 disable TestAcidGuarantees 2023-09-29T06:13:30,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=36, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=TestAcidGuarantees 2023-09-29T06:13:30,322 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968010322"}]},"ts":"1695968010322"} 2023-09-29T06:13:30,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=36 2023-09-29T06:13:30,324 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2023-09-29T06:13:30,326 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(293): Set TestAcidGuarantees to state=DISABLING 2023-09-29T06:13:30,328 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=1291e331f06575ebce0d38fd8ca0d9de, UNASSIGN}] 2023-09-29T06:13:30,329 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=1291e331f06575ebce0d38fd8ca0d9de, UNASSIGN 2023-09-29T06:13:30,330 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=37 updating hbase:meta row=1291e331f06575ebce0d38fd8ca0d9de, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:30,332 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE; CloseRegionProcedure 1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138}] 2023-09-29T06:13:30,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=36 2023-09-29T06:13:30,493 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(119): Close 1291e331f06575ebce0d38fd8ca0d9de 2023-09-29T06:13:30,493 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1668): Closing 1291e331f06575ebce0d38fd8ca0d9de, disabling compactions & flushes 2023-09-29T06:13:30,493 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1690): Closing region TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:30,493 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1711): Waiting without time limit for close lock on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:30,493 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1778): Acquired close lock on TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. after waiting 0 ms 2023-09-29T06:13:30,493 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1788): Updates disabled for region TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:30,493 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(2819): Flushing 1291e331f06575ebce0d38fd8ca0d9de 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2023-09-29T06:13:30,493 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=A 2023-09-29T06:13:30,494 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:30,494 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=B 2023-09-29T06:13:30,494 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:30,494 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1291e331f06575ebce0d38fd8ca0d9de, store=C 2023-09-29T06:13:30,494 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:30,500 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/e1e66f3a58b64996b83358799ea9a220 is 50, key is test_row_0/A:col10/1695968010293/Put/seqid=0 2023-09-29T06:13:30,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=36 2023-09-29T06:13:30,905 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=598 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/e1e66f3a58b64996b83358799ea9a220 2023-09-29T06:13:30,915 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/653685c28de24fc8a6dcf1b7663c2fcc is 50, key is test_row_0/B:col10/1695968010293/Put/seqid=0 2023-09-29T06:13:30,921 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=598 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/653685c28de24fc8a6dcf1b7663c2fcc 2023-09-29T06:13:30,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=36 2023-09-29T06:13:30,931 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/f0dc23fd72fd4380bcc41c34dbbb082a is 50, key is test_row_0/C:col10/1695968010293/Put/seqid=0 2023-09-29T06:13:31,336 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=598 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/f0dc23fd72fd4380bcc41c34dbbb082a 2023-09-29T06:13:31,343 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/A/e1e66f3a58b64996b83358799ea9a220 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/e1e66f3a58b64996b83358799ea9a220 2023-09-29T06:13:31,349 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/e1e66f3a58b64996b83358799ea9a220, entries=50, sequenceid=598, filesize=7.2 K 2023-09-29T06:13:31,350 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/B/653685c28de24fc8a6dcf1b7663c2fcc as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/653685c28de24fc8a6dcf1b7663c2fcc 2023-09-29T06:13:31,358 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/653685c28de24fc8a6dcf1b7663c2fcc, entries=50, sequenceid=598, filesize=7.2 K 2023-09-29T06:13:31,360 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/.tmp/C/f0dc23fd72fd4380bcc41c34dbbb082a as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/f0dc23fd72fd4380bcc41c34dbbb082a 2023-09-29T06:13:31,366 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/f0dc23fd72fd4380bcc41c34dbbb082a, entries=50, sequenceid=598, filesize=7.2 K 2023-09-29T06:13:31,367 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(3022): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 1291e331f06575ebce0d38fd8ca0d9de in 874ms, sequenceid=598, compaction requested=false 2023-09-29T06:13:31,399 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/f3f958725f1740dab741720d3eb189ad, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/5f6ff41a29b549c8b74e3642fe2fbe3f, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/c6a8cd8f6dd6454688217395da8d0e35, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/935432a161704c2b828ac0509a6523c2, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/91ad6e696e5e45e7a21e5f24cc86fce9, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/cda5389353e1479a923e096305a78960, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/d73ebafa2a5a4671b809f1d250d46c66, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/d70a7386a8d742848ee85a3b29018a78, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/ba0ab8d9e3ba4a3997ce33c74fd0f0f9, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/d03fde6465284b888486d9e5f8610c74, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/b081b9464fe84a27ad21fdbd94ac614d, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/3003e4dbd3ba4ba7b2da9e8a40cebeae, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/6b67e203054b46209699512d71ab24a2, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/030789cb8ca74e25b544c375b0dc3038, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/e1d644c0965f4fa9a54ded5b8da40c0d, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/cc83e59996bb458f80c6fb040c6ac977, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/a38030f69f774fdabc0c907151a01784, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/7bbfa799a034448eb77e88ec394c0b97, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/4835b823b1f44749952d47bb657b1fb4, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/06356818e5d9423c9be67b38af6c83a4, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/38bdaa14502c46c2a7e24294978e32dd, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/abe22ae6e20c41d1b298113861429f37, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/3ea27524b4274891a6cd847f529cfa76, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/7b17438bba3f4693904ac00e9a5e871b, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/05ec72ccf0cf4941a48fa91cc3b67cf6, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/9470ee2994744772b820295bd9f409d1, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/3140377bcca74f89853bb266eb801def, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/6d31eb78e62b49ea86f6548379c26831, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/e3f6d5929faf43bb81e01f70b9d8bb8b, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/4a852996bd5e4756b823e05ff93648b5, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/aad8e39a39904d5cafdeeb0f4585d6cd, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/13d4415cb3a2450c901b16d417402092, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/06b6f949408b4e95be3e89d4fb83b0a2, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/9aebd9b0a4eb47c7bd69bf659114f356, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/f8bd2e7453b042aaa30880903984fcff, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/1706e27398ae44acb2d4c4cf7e1a1e2c, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/ce52e15280e34c62a2b277ede1a888d9, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/d4fe7d3da5db4623b1a54af454ebfc89, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/80265bc85904496d9ff18587eb780290, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/ede78ea2cfc049bf89716fad95f1929a, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/a148d1d401d04fddbd131e4af96cc14a, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/e90bbd9f5f974e518d8c16ea5266a85b] to archive 2023-09-29T06:13:31,403 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2023-09-29T06:13:31,410 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/f3f958725f1740dab741720d3eb189ad to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/f3f958725f1740dab741720d3eb189ad 2023-09-29T06:13:31,412 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/5f6ff41a29b549c8b74e3642fe2fbe3f to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/5f6ff41a29b549c8b74e3642fe2fbe3f 2023-09-29T06:13:31,414 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/c6a8cd8f6dd6454688217395da8d0e35 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/c6a8cd8f6dd6454688217395da8d0e35 2023-09-29T06:13:31,416 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/935432a161704c2b828ac0509a6523c2 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/935432a161704c2b828ac0509a6523c2 2023-09-29T06:13:31,418 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/91ad6e696e5e45e7a21e5f24cc86fce9 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/91ad6e696e5e45e7a21e5f24cc86fce9 2023-09-29T06:13:31,420 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/cda5389353e1479a923e096305a78960 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/cda5389353e1479a923e096305a78960 2023-09-29T06:13:31,422 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/d73ebafa2a5a4671b809f1d250d46c66 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/d73ebafa2a5a4671b809f1d250d46c66 2023-09-29T06:13:31,424 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/d70a7386a8d742848ee85a3b29018a78 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/d70a7386a8d742848ee85a3b29018a78 2023-09-29T06:13:31,426 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/ba0ab8d9e3ba4a3997ce33c74fd0f0f9 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/ba0ab8d9e3ba4a3997ce33c74fd0f0f9 2023-09-29T06:13:31,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=36 2023-09-29T06:13:31,428 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/d03fde6465284b888486d9e5f8610c74 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/d03fde6465284b888486d9e5f8610c74 2023-09-29T06:13:31,430 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/b081b9464fe84a27ad21fdbd94ac614d to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/b081b9464fe84a27ad21fdbd94ac614d 2023-09-29T06:13:31,431 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/3003e4dbd3ba4ba7b2da9e8a40cebeae to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/3003e4dbd3ba4ba7b2da9e8a40cebeae 2023-09-29T06:13:31,432 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/6b67e203054b46209699512d71ab24a2 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/6b67e203054b46209699512d71ab24a2 2023-09-29T06:13:31,434 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/030789cb8ca74e25b544c375b0dc3038 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/030789cb8ca74e25b544c375b0dc3038 2023-09-29T06:13:31,435 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/e1d644c0965f4fa9a54ded5b8da40c0d to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/e1d644c0965f4fa9a54ded5b8da40c0d 2023-09-29T06:13:31,437 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/cc83e59996bb458f80c6fb040c6ac977 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/cc83e59996bb458f80c6fb040c6ac977 2023-09-29T06:13:31,439 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/a38030f69f774fdabc0c907151a01784 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/a38030f69f774fdabc0c907151a01784 2023-09-29T06:13:31,441 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/7bbfa799a034448eb77e88ec394c0b97 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/7bbfa799a034448eb77e88ec394c0b97 2023-09-29T06:13:31,442 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/4835b823b1f44749952d47bb657b1fb4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/4835b823b1f44749952d47bb657b1fb4 2023-09-29T06:13:31,444 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/06356818e5d9423c9be67b38af6c83a4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/06356818e5d9423c9be67b38af6c83a4 2023-09-29T06:13:31,446 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/38bdaa14502c46c2a7e24294978e32dd to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/38bdaa14502c46c2a7e24294978e32dd 2023-09-29T06:13:31,448 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/abe22ae6e20c41d1b298113861429f37 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/abe22ae6e20c41d1b298113861429f37 2023-09-29T06:13:31,450 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/3ea27524b4274891a6cd847f529cfa76 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/3ea27524b4274891a6cd847f529cfa76 2023-09-29T06:13:31,451 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/7b17438bba3f4693904ac00e9a5e871b to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/7b17438bba3f4693904ac00e9a5e871b 2023-09-29T06:13:31,452 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/05ec72ccf0cf4941a48fa91cc3b67cf6 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/05ec72ccf0cf4941a48fa91cc3b67cf6 2023-09-29T06:13:31,454 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/9470ee2994744772b820295bd9f409d1 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/9470ee2994744772b820295bd9f409d1 2023-09-29T06:13:31,455 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/3140377bcca74f89853bb266eb801def to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/3140377bcca74f89853bb266eb801def 2023-09-29T06:13:31,456 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/6d31eb78e62b49ea86f6548379c26831 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/6d31eb78e62b49ea86f6548379c26831 2023-09-29T06:13:31,458 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/e3f6d5929faf43bb81e01f70b9d8bb8b to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/e3f6d5929faf43bb81e01f70b9d8bb8b 2023-09-29T06:13:31,459 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/4a852996bd5e4756b823e05ff93648b5 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/4a852996bd5e4756b823e05ff93648b5 2023-09-29T06:13:31,460 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/aad8e39a39904d5cafdeeb0f4585d6cd to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/aad8e39a39904d5cafdeeb0f4585d6cd 2023-09-29T06:13:31,461 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/13d4415cb3a2450c901b16d417402092 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/13d4415cb3a2450c901b16d417402092 2023-09-29T06:13:31,463 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/06b6f949408b4e95be3e89d4fb83b0a2 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/06b6f949408b4e95be3e89d4fb83b0a2 2023-09-29T06:13:31,464 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/9aebd9b0a4eb47c7bd69bf659114f356 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/9aebd9b0a4eb47c7bd69bf659114f356 2023-09-29T06:13:31,466 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/f8bd2e7453b042aaa30880903984fcff to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/f8bd2e7453b042aaa30880903984fcff 2023-09-29T06:13:31,468 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/1706e27398ae44acb2d4c4cf7e1a1e2c to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/1706e27398ae44acb2d4c4cf7e1a1e2c 2023-09-29T06:13:31,469 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/ce52e15280e34c62a2b277ede1a888d9 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/ce52e15280e34c62a2b277ede1a888d9 2023-09-29T06:13:31,471 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/d4fe7d3da5db4623b1a54af454ebfc89 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/d4fe7d3da5db4623b1a54af454ebfc89 2023-09-29T06:13:31,472 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/80265bc85904496d9ff18587eb780290 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/80265bc85904496d9ff18587eb780290 2023-09-29T06:13:31,473 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/ede78ea2cfc049bf89716fad95f1929a to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/ede78ea2cfc049bf89716fad95f1929a 2023-09-29T06:13:31,474 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/a148d1d401d04fddbd131e4af96cc14a to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/a148d1d401d04fddbd131e4af96cc14a 2023-09-29T06:13:31,475 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/e90bbd9f5f974e518d8c16ea5266a85b to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/e90bbd9f5f974e518d8c16ea5266a85b 2023-09-29T06:13:31,510 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/5a1257a4902e435099f81cce4f0c7df9, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/2304f50233a444ccbb60f3b2de131c6f, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/967e0faf18684a8e9fd4cf8b4ec601fe, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/04ee45a0174b464fa2eaaa05e9b4dec7, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/60c5c22f85a24de18be368dfff488055, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/eff8c7cd4456421388b638a1ddd0bfe6, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/69c095194f404f9f976b50c0d70af93a, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/703be8a3ae924bc4aae6f13f344f8eae, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/578f75685c56486786749cf5b272a3aa, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/db46cd714ab44c03a88925e24c5929a1, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/4bacee4e8d0f401d8445b0d2f9459187, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/763ac22df391492e9777e19ab4050d7e, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/2d8a1971ac6848489cf4366da08a4bc8, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/a05d7d2fe03045b8a6aefc8cdb56ab65, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/3ab427fd69d5438fb058501d343cdbfb, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/0eea780c66614ebb9ca0e00bee66463b, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/1d3b7102ec624db196f821aa39dd5f02, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/42aa62ce42534ddd84a4d299ef8dcf79, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/69a060142b6a42ddb01e8e01dd514e20, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/113365189067452ead856109076399c0, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/38c54a91d4e34443a1bc6a66b721a4b6, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/8c6be74f0d9542a89050871cdf486be5, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/74085fa665df4af88d24881738b174be, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/2c8dc76ec4ec44379bfa8fd4cdcf682c, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/017af24239a4480ca217dfc927b84e4f, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/e6def42d2a0f42cf940009c986febdb5, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/3927ce7be2aa43fe9bc745376e11e66b, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/02418b3fd8ce4cb28ce440a0a6503464, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/55cb5c428dfb44788fe0dece9a36a9a0, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/e79c5123683b4c57b9715dae4a062a26, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/da93a948772342edb8e93a6329cc7455, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/f14a987c2aca43c4be9f2f45ad72cecb, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/611974814589440abcbfe7fe80c73d54, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/31cf797c975749b0918f22b71066ee01, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/a8f386a9505e4017a1b3e4e45a275f9e, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/4d1c6ef96c0b488ca27ad9ad5defe599, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/637580ed3836447ebb713325f608c691, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/889b98d927c6432cb69cb1db95d1c01e, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/ac14ae6b9c7945cb911e3daf8e882fbd, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/469e52dce4d74160989cd6b68a230c09, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/506cbd5cd00040bf864e31232f391701, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/0ad4b0b197c0440cb9e25584001a3d7b] to archive 2023-09-29T06:13:31,511 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2023-09-29T06:13:31,514 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/5a1257a4902e435099f81cce4f0c7df9 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/5a1257a4902e435099f81cce4f0c7df9 2023-09-29T06:13:31,515 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/2304f50233a444ccbb60f3b2de131c6f to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/2304f50233a444ccbb60f3b2de131c6f 2023-09-29T06:13:31,517 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/967e0faf18684a8e9fd4cf8b4ec601fe to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/967e0faf18684a8e9fd4cf8b4ec601fe 2023-09-29T06:13:31,518 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/04ee45a0174b464fa2eaaa05e9b4dec7 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/04ee45a0174b464fa2eaaa05e9b4dec7 2023-09-29T06:13:31,520 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/60c5c22f85a24de18be368dfff488055 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/60c5c22f85a24de18be368dfff488055 2023-09-29T06:13:31,521 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/eff8c7cd4456421388b638a1ddd0bfe6 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/eff8c7cd4456421388b638a1ddd0bfe6 2023-09-29T06:13:31,522 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/69c095194f404f9f976b50c0d70af93a to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/69c095194f404f9f976b50c0d70af93a 2023-09-29T06:13:31,523 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/703be8a3ae924bc4aae6f13f344f8eae to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/703be8a3ae924bc4aae6f13f344f8eae 2023-09-29T06:13:31,525 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/578f75685c56486786749cf5b272a3aa to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/578f75685c56486786749cf5b272a3aa 2023-09-29T06:13:31,526 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/db46cd714ab44c03a88925e24c5929a1 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/db46cd714ab44c03a88925e24c5929a1 2023-09-29T06:13:31,527 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/4bacee4e8d0f401d8445b0d2f9459187 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/4bacee4e8d0f401d8445b0d2f9459187 2023-09-29T06:13:31,529 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/763ac22df391492e9777e19ab4050d7e to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/763ac22df391492e9777e19ab4050d7e 2023-09-29T06:13:31,530 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/2d8a1971ac6848489cf4366da08a4bc8 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/2d8a1971ac6848489cf4366da08a4bc8 2023-09-29T06:13:31,531 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/a05d7d2fe03045b8a6aefc8cdb56ab65 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/a05d7d2fe03045b8a6aefc8cdb56ab65 2023-09-29T06:13:31,532 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/3ab427fd69d5438fb058501d343cdbfb to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/3ab427fd69d5438fb058501d343cdbfb 2023-09-29T06:13:31,533 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/0eea780c66614ebb9ca0e00bee66463b to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/0eea780c66614ebb9ca0e00bee66463b 2023-09-29T06:13:31,535 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/1d3b7102ec624db196f821aa39dd5f02 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/1d3b7102ec624db196f821aa39dd5f02 2023-09-29T06:13:31,536 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/42aa62ce42534ddd84a4d299ef8dcf79 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/42aa62ce42534ddd84a4d299ef8dcf79 2023-09-29T06:13:31,537 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/69a060142b6a42ddb01e8e01dd514e20 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/69a060142b6a42ddb01e8e01dd514e20 2023-09-29T06:13:31,539 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/113365189067452ead856109076399c0 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/113365189067452ead856109076399c0 2023-09-29T06:13:31,541 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/38c54a91d4e34443a1bc6a66b721a4b6 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/38c54a91d4e34443a1bc6a66b721a4b6 2023-09-29T06:13:31,542 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/8c6be74f0d9542a89050871cdf486be5 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/8c6be74f0d9542a89050871cdf486be5 2023-09-29T06:13:31,545 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/74085fa665df4af88d24881738b174be to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/74085fa665df4af88d24881738b174be 2023-09-29T06:13:31,547 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/2c8dc76ec4ec44379bfa8fd4cdcf682c to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/2c8dc76ec4ec44379bfa8fd4cdcf682c 2023-09-29T06:13:31,548 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/017af24239a4480ca217dfc927b84e4f to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/017af24239a4480ca217dfc927b84e4f 2023-09-29T06:13:31,550 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/e6def42d2a0f42cf940009c986febdb5 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/e6def42d2a0f42cf940009c986febdb5 2023-09-29T06:13:31,551 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/3927ce7be2aa43fe9bc745376e11e66b to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/3927ce7be2aa43fe9bc745376e11e66b 2023-09-29T06:13:31,553 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/02418b3fd8ce4cb28ce440a0a6503464 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/02418b3fd8ce4cb28ce440a0a6503464 2023-09-29T06:13:31,554 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/55cb5c428dfb44788fe0dece9a36a9a0 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/55cb5c428dfb44788fe0dece9a36a9a0 2023-09-29T06:13:31,556 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/e79c5123683b4c57b9715dae4a062a26 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/e79c5123683b4c57b9715dae4a062a26 2023-09-29T06:13:31,557 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/da93a948772342edb8e93a6329cc7455 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/da93a948772342edb8e93a6329cc7455 2023-09-29T06:13:31,559 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/f14a987c2aca43c4be9f2f45ad72cecb to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/f14a987c2aca43c4be9f2f45ad72cecb 2023-09-29T06:13:31,560 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/611974814589440abcbfe7fe80c73d54 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/611974814589440abcbfe7fe80c73d54 2023-09-29T06:13:31,561 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/31cf797c975749b0918f22b71066ee01 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/31cf797c975749b0918f22b71066ee01 2023-09-29T06:13:31,563 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/a8f386a9505e4017a1b3e4e45a275f9e to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/a8f386a9505e4017a1b3e4e45a275f9e 2023-09-29T06:13:31,564 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/4d1c6ef96c0b488ca27ad9ad5defe599 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/4d1c6ef96c0b488ca27ad9ad5defe599 2023-09-29T06:13:31,565 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/637580ed3836447ebb713325f608c691 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/637580ed3836447ebb713325f608c691 2023-09-29T06:13:31,567 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/889b98d927c6432cb69cb1db95d1c01e to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/889b98d927c6432cb69cb1db95d1c01e 2023-09-29T06:13:31,569 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/ac14ae6b9c7945cb911e3daf8e882fbd to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/ac14ae6b9c7945cb911e3daf8e882fbd 2023-09-29T06:13:31,570 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/469e52dce4d74160989cd6b68a230c09 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/469e52dce4d74160989cd6b68a230c09 2023-09-29T06:13:31,571 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/506cbd5cd00040bf864e31232f391701 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/506cbd5cd00040bf864e31232f391701 2023-09-29T06:13:31,573 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/0ad4b0b197c0440cb9e25584001a3d7b to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/0ad4b0b197c0440cb9e25584001a3d7b 2023-09-29T06:13:31,577 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/ac24b6cd9df64fd9afa0d4bb1336af20, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/f6782c14c6df4083a91bb4f738ca167a, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/b00303588bd14d21860f5ead4cfaf899, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/b60a1211e07149c8ab65dafad473c22a, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/30a7353119a84c3bb52780ea67071fc0, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/ff4bf8adf82743c592d21169d667e216, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/ce3390af828e4e4e815984f24c1a20d8, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/c999424ef138458c96260b379a430e94, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/fc0c22d481bc4b75a65451a5739385b8, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/e33f6a4efb36431198f0ef5ab7ace513, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/2252230c61bf44749d4d0b3def1ef74e, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/e95a249b75bb41d7a6d3160397fc845a, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/e0105ac4698844b4a818e17907d613b1, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/ba5a1e2932cb44a193cd52e4f667791d, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/3078da1ea1684bd48536ca94fc6500f8, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/a8999a0b87a44794ac586a88ffb4b124, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/e190d90f852d49928d8995003b6ed7e9, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/f200b728a9134a1fbeeb50a6391aeaa1, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/8955663353cf4bcfa6511397436d9731, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/1de1b459300b418ba856a997e70466c2, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/f6393955edaf43de80a7b07f64bb19f3, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/d65420780b02414a884196fb489e2792, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/7deb69431536494c9c7d98896ae8d456, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/64c6e7035a744473ae2bc9406622c96b, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/7de828c345454202b934a75bbb6d2451, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/268074203da74613ac5edb040b091e12, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/fea8317613ec48d9a42058ef39d1b830, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/7c09918d63564284a3d8885530d6b18f, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/b36fd3ea48e74b798497df97a19b0e90, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/a58547338ae14bad8be5d5ebfc27ca02, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/1bb3b1f26ea84029ba07aad4338133d8, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/b551bdf098654f329e51dd6757ea29d2, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/9393dbc4a0044b72b0de077758cdbbfb, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/0dfd10fc97d145dd9f3d0a25220e7fc5, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/88906a25f53c4a6f83dc4f571d4129bc, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/bf455c93a87f4d31bf6f9971c06b3c29, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/2e230bcc8fb14029baa49a9ffbe7015d, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/dc99aa22cbcc4c938ff14991de63f517, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/bb17ec84de814123b3b9aaa4326b6aee, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/f1424248f43940e69f7cce25d5d37c25] to archive 2023-09-29T06:13:31,579 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2023-09-29T06:13:31,580 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/ac24b6cd9df64fd9afa0d4bb1336af20 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/ac24b6cd9df64fd9afa0d4bb1336af20 2023-09-29T06:13:31,582 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/f6782c14c6df4083a91bb4f738ca167a to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/f6782c14c6df4083a91bb4f738ca167a 2023-09-29T06:13:31,583 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/b00303588bd14d21860f5ead4cfaf899 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/b00303588bd14d21860f5ead4cfaf899 2023-09-29T06:13:31,584 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/b60a1211e07149c8ab65dafad473c22a to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/b60a1211e07149c8ab65dafad473c22a 2023-09-29T06:13:31,586 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/30a7353119a84c3bb52780ea67071fc0 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/30a7353119a84c3bb52780ea67071fc0 2023-09-29T06:13:31,587 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/ff4bf8adf82743c592d21169d667e216 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/ff4bf8adf82743c592d21169d667e216 2023-09-29T06:13:31,588 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/ce3390af828e4e4e815984f24c1a20d8 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/ce3390af828e4e4e815984f24c1a20d8 2023-09-29T06:13:31,590 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/c999424ef138458c96260b379a430e94 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/c999424ef138458c96260b379a430e94 2023-09-29T06:13:31,592 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/fc0c22d481bc4b75a65451a5739385b8 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/fc0c22d481bc4b75a65451a5739385b8 2023-09-29T06:13:31,594 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/e33f6a4efb36431198f0ef5ab7ace513 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/e33f6a4efb36431198f0ef5ab7ace513 2023-09-29T06:13:31,595 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/2252230c61bf44749d4d0b3def1ef74e to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/2252230c61bf44749d4d0b3def1ef74e 2023-09-29T06:13:31,596 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/e95a249b75bb41d7a6d3160397fc845a to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/e95a249b75bb41d7a6d3160397fc845a 2023-09-29T06:13:31,598 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/e0105ac4698844b4a818e17907d613b1 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/e0105ac4698844b4a818e17907d613b1 2023-09-29T06:13:31,599 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/ba5a1e2932cb44a193cd52e4f667791d to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/ba5a1e2932cb44a193cd52e4f667791d 2023-09-29T06:13:31,600 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/3078da1ea1684bd48536ca94fc6500f8 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/3078da1ea1684bd48536ca94fc6500f8 2023-09-29T06:13:31,602 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/a8999a0b87a44794ac586a88ffb4b124 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/a8999a0b87a44794ac586a88ffb4b124 2023-09-29T06:13:31,604 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/e190d90f852d49928d8995003b6ed7e9 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/e190d90f852d49928d8995003b6ed7e9 2023-09-29T06:13:31,606 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/f200b728a9134a1fbeeb50a6391aeaa1 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/f200b728a9134a1fbeeb50a6391aeaa1 2023-09-29T06:13:31,608 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/8955663353cf4bcfa6511397436d9731 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/8955663353cf4bcfa6511397436d9731 2023-09-29T06:13:31,609 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/1de1b459300b418ba856a997e70466c2 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/1de1b459300b418ba856a997e70466c2 2023-09-29T06:13:31,610 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/f6393955edaf43de80a7b07f64bb19f3 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/f6393955edaf43de80a7b07f64bb19f3 2023-09-29T06:13:31,612 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/d65420780b02414a884196fb489e2792 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/d65420780b02414a884196fb489e2792 2023-09-29T06:13:31,613 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/7deb69431536494c9c7d98896ae8d456 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/7deb69431536494c9c7d98896ae8d456 2023-09-29T06:13:31,614 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/64c6e7035a744473ae2bc9406622c96b to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/64c6e7035a744473ae2bc9406622c96b 2023-09-29T06:13:31,615 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/7de828c345454202b934a75bbb6d2451 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/7de828c345454202b934a75bbb6d2451 2023-09-29T06:13:31,616 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/268074203da74613ac5edb040b091e12 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/268074203da74613ac5edb040b091e12 2023-09-29T06:13:31,618 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/fea8317613ec48d9a42058ef39d1b830 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/fea8317613ec48d9a42058ef39d1b830 2023-09-29T06:13:31,619 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/7c09918d63564284a3d8885530d6b18f to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/7c09918d63564284a3d8885530d6b18f 2023-09-29T06:13:31,622 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/b36fd3ea48e74b798497df97a19b0e90 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/b36fd3ea48e74b798497df97a19b0e90 2023-09-29T06:13:31,623 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/a58547338ae14bad8be5d5ebfc27ca02 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/a58547338ae14bad8be5d5ebfc27ca02 2023-09-29T06:13:31,625 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/1bb3b1f26ea84029ba07aad4338133d8 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/1bb3b1f26ea84029ba07aad4338133d8 2023-09-29T06:13:31,626 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/b551bdf098654f329e51dd6757ea29d2 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/b551bdf098654f329e51dd6757ea29d2 2023-09-29T06:13:31,627 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/9393dbc4a0044b72b0de077758cdbbfb to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/9393dbc4a0044b72b0de077758cdbbfb 2023-09-29T06:13:31,629 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/0dfd10fc97d145dd9f3d0a25220e7fc5 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/0dfd10fc97d145dd9f3d0a25220e7fc5 2023-09-29T06:13:31,630 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/88906a25f53c4a6f83dc4f571d4129bc to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/88906a25f53c4a6f83dc4f571d4129bc 2023-09-29T06:13:31,631 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/bf455c93a87f4d31bf6f9971c06b3c29 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/bf455c93a87f4d31bf6f9971c06b3c29 2023-09-29T06:13:31,633 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/2e230bcc8fb14029baa49a9ffbe7015d to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/2e230bcc8fb14029baa49a9ffbe7015d 2023-09-29T06:13:31,634 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/dc99aa22cbcc4c938ff14991de63f517 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/dc99aa22cbcc4c938ff14991de63f517 2023-09-29T06:13:31,636 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/bb17ec84de814123b3b9aaa4326b6aee to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/bb17ec84de814123b3b9aaa4326b6aee 2023-09-29T06:13:31,637 DEBUG [StoreCloser-TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/f1424248f43940e69f7cce25d5d37c25 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/f1424248f43940e69f7cce25d5d37c25 2023-09-29T06:13:31,646 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/recovered.edits/601.seqid, newMaxSeqId=601, maxSeqId=1 2023-09-29T06:13:31,648 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1909): Closed TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de. 2023-09-29T06:13:31,648 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1622): Region close journal for 1291e331f06575ebce0d38fd8ca0d9de: 2023-09-29T06:13:31,650 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(163): Closed 1291e331f06575ebce0d38fd8ca0d9de 2023-09-29T06:13:31,650 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=37 updating hbase:meta row=1291e331f06575ebce0d38fd8ca0d9de, regionState=CLOSED 2023-09-29T06:13:31,653 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=38, resume processing ppid=37 2023-09-29T06:13:31,653 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=38, ppid=37, state=SUCCESS; CloseRegionProcedure 1291e331f06575ebce0d38fd8ca0d9de, server=jenkins-hbase3.apache.org,39783,1695967978138 in 1.3190 sec 2023-09-29T06:13:31,655 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=37, resume processing ppid=36 2023-09-29T06:13:31,655 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=37, ppid=36, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=1291e331f06575ebce0d38fd8ca0d9de, UNASSIGN in 1.3250 sec 2023-09-29T06:13:31,656 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968011655"}]},"ts":"1695968011655"} 2023-09-29T06:13:31,657 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2023-09-29T06:13:31,660 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(305): Set TestAcidGuarantees to state=DISABLED 2023-09-29T06:13:31,662 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=36, state=SUCCESS; DisableTableProcedure table=TestAcidGuarantees in 1.3490 sec 2023-09-29T06:13:32,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=36 2023-09-29T06:13:32,428 INFO [Listener at localhost/42843 {}] client.HBaseAdmin$TableFuture(3677): Operation: DISABLE, Table Name: default:TestAcidGuarantees, procId: 36 completed 2023-09-29T06:13:32,433 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$5(2448): Client=jenkins//172.31.12.81 delete TestAcidGuarantees 2023-09-29T06:13:32,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=39, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=TestAcidGuarantees 2023-09-29T06:13:32,443 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=39, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2023-09-29T06:13:32,444 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=39, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2023-09-29T06:13:32,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=39 2023-09-29T06:13:32,449 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de 2023-09-29T06:13:32,455 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A, FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B, FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C, FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/recovered.edits] 2023-09-29T06:13:32,460 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/bfbd24a11b3446ecbd3245bc9057f743 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/bfbd24a11b3446ecbd3245bc9057f743 2023-09-29T06:13:32,462 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/e1e66f3a58b64996b83358799ea9a220 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/A/e1e66f3a58b64996b83358799ea9a220 2023-09-29T06:13:32,467 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/61013a0b109e4a39a82a964033b7f23e to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/61013a0b109e4a39a82a964033b7f23e 2023-09-29T06:13:32,469 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/653685c28de24fc8a6dcf1b7663c2fcc to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/B/653685c28de24fc8a6dcf1b7663c2fcc 2023-09-29T06:13:32,473 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/237cc160f985423c96d608feef08e614 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/237cc160f985423c96d608feef08e614 2023-09-29T06:13:32,475 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/f0dc23fd72fd4380bcc41c34dbbb082a to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/C/f0dc23fd72fd4380bcc41c34dbbb082a 2023-09-29T06:13:32,479 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/recovered.edits/601.seqid to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de/recovered.edits/601.seqid 2023-09-29T06:13:32,480 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1291e331f06575ebce0d38fd8ca0d9de 2023-09-29T06:13:32,480 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(293): Archived TestAcidGuarantees regions 2023-09-29T06:13:32,490 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=39, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2023-09-29T06:13:32,514 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2023-09-29T06:13:32,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=39 2023-09-29T06:13:32,567 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(387): Removing 'TestAcidGuarantees' descriptor. 2023-09-29T06:13:32,569 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=39, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2023-09-29T06:13:32,569 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(377): Removing 'TestAcidGuarantees' from region states. 2023-09-29T06:13:32,569 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1695968012569"}]},"ts":"9223372036854775807"} 2023-09-29T06:13:32,572 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-09-29T06:13:32,572 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 1291e331f06575ebce0d38fd8ca0d9de, NAME => 'TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de.', STARTKEY => '', ENDKEY => ''}] 2023-09-29T06:13:32,572 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(381): Marking 'TestAcidGuarantees' as deleted. 2023-09-29T06:13:32,572 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1695968012572"}]},"ts":"9223372036854775807"} 2023-09-29T06:13:32,574 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1678): Deleted table TestAcidGuarantees state from META 2023-09-29T06:13:32,577 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(127): Finished pid=39, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2023-09-29T06:13:32,578 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=39, state=SUCCESS; DeleteTableProcedure table=TestAcidGuarantees in 143 msec 2023-09-29T06:13:32,631 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-09-29T06:13:32,634 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:52710, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-09-29T06:13:32,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=39 2023-09-29T06:13:32,749 INFO [Listener at localhost/42843 {}] client.HBaseAdmin$TableFuture(3677): Operation: DELETE, Table Name: default:TestAcidGuarantees, procId: 39 completed 2023-09-29T06:13:32,773 INFO [Listener at localhost/42843 {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithAdaptivePolicy#testMixedAtomicity Thread=220 (was 204) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RSProcedureDispatcher-pool-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x12d7eb4f-shared-pool-7 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x12d7eb4f-shared-pool-4 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS-EventLoopGroup-1-3 org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:209) org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:202) org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:306) org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:363) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Timer for 'HBase' metrics system java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: HFileArchiver-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x12d7eb4f-shared-pool-5 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x12d7eb4f-shared-pool-6 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-975914720_18 at /127.0.0.1:37502 [Waiting for operation #3] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS:0;jenkins-hbase3:39783-shortCompactions-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:549) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=570 (was 478) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=659 (was 581) - SystemLoadAverage LEAK? -, ProcessCount=171 (was 171), AvailableMemoryMB=2036 (was 2568) 2023-09-29T06:13:32,789 INFO [Listener at localhost/42843 {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithAdaptivePolicy#testMobMixedAtomicity Thread=220, OpenFileDescriptor=570, MaxFileDescriptor=60000, SystemLoadAverage=659, ProcessCount=171, AvailableMemoryMB=2035 2023-09-29T06:13:32,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] util.TableDescriptorChecker(342): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2023-09-29T06:13:32,792 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'ADAPTIVE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-29T06:13:32,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=40, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestAcidGuarantees 2023-09-29T06:13:32,795 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=40, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2023-09-29T06:13:32,795 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:13:32,795 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 40 2023-09-29T06:13:32,796 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=40, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-29T06:13:32,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=40 2023-09-29T06:13:32,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=40 2023-09-29T06:13:33,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=40 2023-09-29T06:13:33,208 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'ADAPTIVE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a 2023-09-29T06:13:33,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=40 2023-09-29T06:13:33,615 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(888): Instantiated TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:13:33,615 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1668): Closing 1668fccb65cfe5ce16ea12b0cc0df5e4, disabling compactions & flushes 2023-09-29T06:13:33,615 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1690): Closing region TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:33,615 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:33,616 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. after waiting 0 ms 2023-09-29T06:13:33,616 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:33,616 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1909): Closed TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:33,616 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1622): Region close journal for 1668fccb65cfe5ce16ea12b0cc0df5e4: 2023-09-29T06:13:33,617 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=40, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2023-09-29T06:13:33,617 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1695968013617"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1695968013617"}]},"ts":"1695968013617"} 2023-09-29T06:13:33,619 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-29T06:13:33,620 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=40, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-29T06:13:33,620 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968013620"}]},"ts":"1695968013620"} 2023-09-29T06:13:33,621 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2023-09-29T06:13:33,628 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=1668fccb65cfe5ce16ea12b0cc0df5e4, ASSIGN}] 2023-09-29T06:13:33,630 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=1668fccb65cfe5ce16ea12b0cc0df5e4, ASSIGN 2023-09-29T06:13:33,631 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1668fccb65cfe5ce16ea12b0cc0df5e4, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,39783,1695967978138; forceNewPlan=false, retain=false 2023-09-29T06:13:33,782 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=41 updating hbase:meta row=1668fccb65cfe5ce16ea12b0cc0df5e4, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:33,784 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=42, ppid=41, state=RUNNABLE; OpenRegionProcedure 1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138}] 2023-09-29T06:13:33,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=40 2023-09-29T06:13:33,942 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(130): Open TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:33,942 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7254): Opening region: {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} 2023-09-29T06:13:33,942 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:33,942 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(888): Instantiated TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:13:33,942 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7296): checking encryption for 1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:33,942 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7299): checking classloading for 1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:33,944 INFO [StoreOpener-1668fccb65cfe5ce16ea12b0cc0df5e4-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:33,945 INFO [StoreOpener-1668fccb65cfe5ce16ea12b0cc0df5e4-1 {}] regionserver.CompactingMemStore(121): Store=A, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2023-09-29T06:13:33,945 INFO [StoreOpener-1668fccb65cfe5ce16ea12b0cc0df5e4-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1668fccb65cfe5ce16ea12b0cc0df5e4 columnFamilyName A 2023-09-29T06:13:33,946 DEBUG [StoreOpener-1668fccb65cfe5ce16ea12b0cc0df5e4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:13:33,946 INFO [StoreOpener-1668fccb65cfe5ce16ea12b0cc0df5e4-1 {}] regionserver.HStore(324): Store=1668fccb65cfe5ce16ea12b0cc0df5e4/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:13:33,947 INFO [StoreOpener-1668fccb65cfe5ce16ea12b0cc0df5e4-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:33,948 INFO [StoreOpener-1668fccb65cfe5ce16ea12b0cc0df5e4-1 {}] regionserver.CompactingMemStore(121): Store=B, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2023-09-29T06:13:33,949 INFO [StoreOpener-1668fccb65cfe5ce16ea12b0cc0df5e4-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1668fccb65cfe5ce16ea12b0cc0df5e4 columnFamilyName B 2023-09-29T06:13:33,950 DEBUG [StoreOpener-1668fccb65cfe5ce16ea12b0cc0df5e4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:13:33,950 INFO [StoreOpener-1668fccb65cfe5ce16ea12b0cc0df5e4-1 {}] regionserver.HStore(324): Store=1668fccb65cfe5ce16ea12b0cc0df5e4/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:13:33,950 INFO [StoreOpener-1668fccb65cfe5ce16ea12b0cc0df5e4-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:33,951 INFO [StoreOpener-1668fccb65cfe5ce16ea12b0cc0df5e4-1 {}] regionserver.CompactingMemStore(121): Store=C, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2023-09-29T06:13:33,952 INFO [StoreOpener-1668fccb65cfe5ce16ea12b0cc0df5e4-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1668fccb65cfe5ce16ea12b0cc0df5e4 columnFamilyName C 2023-09-29T06:13:33,952 DEBUG [StoreOpener-1668fccb65cfe5ce16ea12b0cc0df5e4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:13:33,952 INFO [StoreOpener-1668fccb65cfe5ce16ea12b0cc0df5e4-1 {}] regionserver.HStore(324): Store=1668fccb65cfe5ce16ea12b0cc0df5e4/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:13:33,953 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1169): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:33,953 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:33,954 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:33,970 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2023-09-29T06:13:33,971 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1076): writing seq id for 1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:33,973 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-29T06:13:33,974 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): Opened 1668fccb65cfe5ce16ea12b0cc0df5e4; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68534735, jitterRate=0.02124713361263275}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2023-09-29T06:13:33,974 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(992): Region open journal for 1668fccb65cfe5ce16ea12b0cc0df5e4: 2023-09-29T06:13:33,975 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2556): Post open deploy tasks for TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4., pid=42, masterSystemTime=1695968013936 2023-09-29T06:13:33,977 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2583): Finished post open deploy task for TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:33,977 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(158): Opened TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:33,977 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=41 updating hbase:meta row=1668fccb65cfe5ce16ea12b0cc0df5e4, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:33,980 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=42, resume processing ppid=41 2023-09-29T06:13:33,980 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=42, ppid=41, state=SUCCESS; OpenRegionProcedure 1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 in 194 msec 2023-09-29T06:13:33,982 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=41, resume processing ppid=40 2023-09-29T06:13:33,982 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=41, ppid=40, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=1668fccb65cfe5ce16ea12b0cc0df5e4, ASSIGN in 352 msec 2023-09-29T06:13:33,982 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=40, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-29T06:13:33,982 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968013982"}]},"ts":"1695968013982"} 2023-09-29T06:13:33,984 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2023-09-29T06:13:33,987 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=40, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2023-09-29T06:13:33,989 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=40, state=SUCCESS; CreateTableProcedure table=TestAcidGuarantees in 1.1950 sec 2023-09-29T06:13:34,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=40 2023-09-29T06:13:34,903 INFO [Listener at localhost/42843 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: default:TestAcidGuarantees, procId: 40 completed 2023-09-29T06:13:34,904 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x1394947b to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:13:34,910 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@643652b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:13:34,913 DEBUG [Listener at localhost/42843 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:13:34,916 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:41422, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:13:34,917 DEBUG [Listener at localhost/42843 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-29T06:13:34,920 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:52724, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-29T06:13:34,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] util.TableDescriptorChecker(342): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2023-09-29T06:13:34,928 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$13(2700): Client=jenkins//172.31.12.81 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'ADAPTIVE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'ADAPTIVE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-29T06:13:34,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=43, state=RUNNABLE:MODIFY_TABLE_PREPARE; ModifyTableProcedure table=TestAcidGuarantees 2023-09-29T06:13:35,359 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.963 2023-09-29T06:13:35,359 INFO [PEWorker-3 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.999 2023-09-29T06:13:35,364 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=44, ppid=43, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2023-09-29T06:13:35,379 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=1668fccb65cfe5ce16ea12b0cc0df5e4, REOPEN/MOVE}] 2023-09-29T06:13:35,381 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=45, ppid=44, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=1668fccb65cfe5ce16ea12b0cc0df5e4, REOPEN/MOVE 2023-09-29T06:13:35,381 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=45 updating hbase:meta row=1668fccb65cfe5ce16ea12b0cc0df5e4, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:35,383 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=46, ppid=45, state=RUNNABLE; CloseRegionProcedure 1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138}] 2023-09-29T06:13:35,536 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] handler.UnassignRegionHandler(119): Close 1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:35,537 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1668): Closing 1668fccb65cfe5ce16ea12b0cc0df5e4, disabling compactions & flushes 2023-09-29T06:13:35,537 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1690): Closing region TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:35,537 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1711): Waiting without time limit for close lock on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:35,537 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1778): Acquired close lock on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. after waiting 0 ms 2023-09-29T06:13:35,537 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1788): Updates disabled for region TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:35,548 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-09-29T06:13:35,549 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1909): Closed TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:35,549 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1622): Region close journal for 1668fccb65cfe5ce16ea12b0cc0df5e4: 2023-09-29T06:13:35,549 WARN [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegionServer(3735): Not adding moved region record: 1668fccb65cfe5ce16ea12b0cc0df5e4 to self. 2023-09-29T06:13:35,551 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] handler.UnassignRegionHandler(163): Closed 1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:35,551 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=45 updating hbase:meta row=1668fccb65cfe5ce16ea12b0cc0df5e4, regionState=CLOSED 2023-09-29T06:13:35,556 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=46, resume processing ppid=45 2023-09-29T06:13:35,556 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=46, ppid=45, state=SUCCESS; CloseRegionProcedure 1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 in 171 msec 2023-09-29T06:13:35,556 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=45, ppid=44, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1668fccb65cfe5ce16ea12b0cc0df5e4, REOPEN/MOVE; state=CLOSED, location=jenkins-hbase3.apache.org,39783,1695967978138; forceNewPlan=false, retain=true 2023-09-29T06:13:35,707 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=45 updating hbase:meta row=1668fccb65cfe5ce16ea12b0cc0df5e4, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:35,710 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=47, ppid=45, state=RUNNABLE; OpenRegionProcedure 1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138}] 2023-09-29T06:13:35,865 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(130): Open TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:35,865 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7254): Opening region: {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} 2023-09-29T06:13:35,866 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:35,866 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(888): Instantiated TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:13:35,866 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7296): checking encryption for 1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:35,866 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7299): checking classloading for 1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:35,869 INFO [StoreOpener-1668fccb65cfe5ce16ea12b0cc0df5e4-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:35,870 INFO [StoreOpener-1668fccb65cfe5ce16ea12b0cc0df5e4-1 {}] regionserver.CompactingMemStore(121): Store=A, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2023-09-29T06:13:35,878 INFO [StoreOpener-1668fccb65cfe5ce16ea12b0cc0df5e4-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1668fccb65cfe5ce16ea12b0cc0df5e4 columnFamilyName A 2023-09-29T06:13:35,880 DEBUG [StoreOpener-1668fccb65cfe5ce16ea12b0cc0df5e4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:13:35,881 INFO [StoreOpener-1668fccb65cfe5ce16ea12b0cc0df5e4-1 {}] regionserver.HStore(324): Store=1668fccb65cfe5ce16ea12b0cc0df5e4/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:13:35,882 INFO [StoreOpener-1668fccb65cfe5ce16ea12b0cc0df5e4-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:35,882 INFO [StoreOpener-1668fccb65cfe5ce16ea12b0cc0df5e4-1 {}] regionserver.CompactingMemStore(121): Store=B, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2023-09-29T06:13:35,883 INFO [StoreOpener-1668fccb65cfe5ce16ea12b0cc0df5e4-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1668fccb65cfe5ce16ea12b0cc0df5e4 columnFamilyName B 2023-09-29T06:13:35,883 DEBUG [StoreOpener-1668fccb65cfe5ce16ea12b0cc0df5e4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:13:35,883 INFO [StoreOpener-1668fccb65cfe5ce16ea12b0cc0df5e4-1 {}] regionserver.HStore(324): Store=1668fccb65cfe5ce16ea12b0cc0df5e4/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:13:35,884 INFO [StoreOpener-1668fccb65cfe5ce16ea12b0cc0df5e4-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:35,884 INFO [StoreOpener-1668fccb65cfe5ce16ea12b0cc0df5e4-1 {}] regionserver.CompactingMemStore(121): Store=C, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2023-09-29T06:13:35,885 INFO [StoreOpener-1668fccb65cfe5ce16ea12b0cc0df5e4-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1668fccb65cfe5ce16ea12b0cc0df5e4 columnFamilyName C 2023-09-29T06:13:35,885 DEBUG [StoreOpener-1668fccb65cfe5ce16ea12b0cc0df5e4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:13:35,885 INFO [StoreOpener-1668fccb65cfe5ce16ea12b0cc0df5e4-1 {}] regionserver.HStore(324): Store=1668fccb65cfe5ce16ea12b0cc0df5e4/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:13:35,886 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1169): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:35,887 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:35,888 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:35,890 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2023-09-29T06:13:35,891 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1076): writing seq id for 1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:35,892 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): Opened 1668fccb65cfe5ce16ea12b0cc0df5e4; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73508158, jitterRate=0.09535691142082214}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2023-09-29T06:13:35,893 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(992): Region open journal for 1668fccb65cfe5ce16ea12b0cc0df5e4: 2023-09-29T06:13:35,894 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2556): Post open deploy tasks for TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4., pid=47, masterSystemTime=1695968015862 2023-09-29T06:13:35,896 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2583): Finished post open deploy task for TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:35,896 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(158): Opened TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:35,896 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=45 updating hbase:meta row=1668fccb65cfe5ce16ea12b0cc0df5e4, regionState=OPEN, openSeqNum=5, regionLocation=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:35,900 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=47, resume processing ppid=45 2023-09-29T06:13:35,900 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=47, ppid=45, state=SUCCESS; OpenRegionProcedure 1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 in 188 msec 2023-09-29T06:13:35,901 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=45, resume processing ppid=44 2023-09-29T06:13:35,901 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=45, ppid=44, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=1668fccb65cfe5ce16ea12b0cc0df5e4, REOPEN/MOVE in 521 msec 2023-09-29T06:13:35,904 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=44, resume processing ppid=43 2023-09-29T06:13:35,904 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=44, ppid=43, state=SUCCESS; ReopenTableRegionsProcedure table=TestAcidGuarantees in 539 msec 2023-09-29T06:13:35,906 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=43, state=SUCCESS; ModifyTableProcedure table=TestAcidGuarantees in 974 msec 2023-09-29T06:13:35,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=43 2023-09-29T06:13:35,919 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x41c47c71 to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:13:35,925 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68ba9aad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:13:35,926 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x4364a9e3 to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:13:35,932 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f373cd0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:13:35,933 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x55a8701b to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:13:35,942 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13bf07c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:13:35,944 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0d682546 to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:13:35,948 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42e31c15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:13:35,949 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x20caeb3b to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:13:35,954 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c44a871, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:13:35,955 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x1b138f3c to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:13:35,967 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@525dbb0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:13:35,968 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x59bbfd07 to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:13:35,972 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ed6f363, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:13:35,973 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0e69549a to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:13:35,979 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8bcd395, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:13:35,980 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x6583e14a to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:13:35,985 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@218ab4e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:13:36,003 DEBUG [hconnection-0x50acbec8-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:13:36,006 DEBUG [hconnection-0xf9c50ac-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:13:36,010 DEBUG [hconnection-0x6b4a49c-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:13:36,013 DEBUG [hconnection-0x2d83dd54-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:13:36,013 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:41426, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:13:36,014 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:41454, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:13:36,016 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:13:36,018 DEBUG [hconnection-0x3c373fd6-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:13:36,024 DEBUG [hconnection-0x596f50c3-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:13:36,024 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:41440, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:13:36,024 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:41458, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:13:36,026 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:41472, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:13:36,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2023-09-29T06:13:36,035 DEBUG [hconnection-0x3e5ae470-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:13:36,035 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:41478, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:13:36,035 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:13:36,035 DEBUG [hconnection-0x41cade44-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:13:36,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=48 2023-09-29T06:13:36,036 DEBUG [hconnection-0x10f517ee-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:13:36,036 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:41494, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:13:36,038 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:41498, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:13:36,039 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:13:36,039 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:13:36,040 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:41514, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:13:36,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:36,068 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 1668fccb65cfe5ce16ea12b0cc0df5e4 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2023-09-29T06:13:36,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=A 2023-09-29T06:13:36,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:36,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=B 2023-09-29T06:13:36,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:36,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=C 2023-09-29T06:13:36,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:36,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309295cf172ba605c4b39b6c2c5fa9abb417e_1668fccb65cfe5ce16ea12b0cc0df5e4 is 50, key is test_row_0/A:col10/1695968016063/Put/seqid=0 2023-09-29T06:13:36,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=48 2023-09-29T06:13:36,175 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309295cf172ba605c4b39b6c2c5fa9abb417e_1668fccb65cfe5ce16ea12b0cc0df5e4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309295cf172ba605c4b39b6c2c5fa9abb417e_1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:36,176 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/99c7d22c33b94115a55e0f7fcd53e5fc, store: [table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:13:36,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:36,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:36,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41454 deadline: 1695968076121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:36,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968076121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:36,191 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/99c7d22c33b94115a55e0f7fcd53e5fc is 175, key is test_row_0/A:col10/1695968016063/Put/seqid=0 2023-09-29T06:13:36,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:36,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968076194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:36,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:36,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41426 deadline: 1695968076194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:36,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:36,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968076194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:36,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2023-09-29T06:13:36,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:36,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. as already flushing 2023-09-29T06:13:36,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:36,207 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(53): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:36,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:36,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:36,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:36,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41454 deadline: 1695968076291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:36,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:36,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968076295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:36,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:36,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968076302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:36,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:36,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968076303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:36,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:36,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41426 deadline: 1695968076303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:36,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=48 2023-09-29T06:13:36,360 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2023-09-29T06:13:36,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:36,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. as already flushing 2023-09-29T06:13:36,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:36,361 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(53): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:36,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:36,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:36,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:36,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41454 deadline: 1695968076498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:36,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:36,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968076501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:36,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:36,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968076508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:36,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:36,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968076509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:36,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:36,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41426 deadline: 1695968076509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:36,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2023-09-29T06:13:36,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:36,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. as already flushing 2023-09-29T06:13:36,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:36,515 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(53): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:36,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:36,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:36,634 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/99c7d22c33b94115a55e0f7fcd53e5fc 2023-09-29T06:13:36,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=48 2023-09-29T06:13:36,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2023-09-29T06:13:36,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:36,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. as already flushing 2023-09-29T06:13:36,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:36,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(53): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:36,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:36,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:36,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/f812d3ee16bb479cb4faaf3b751a3c52 is 50, key is test_row_0/B:col10/1695968016063/Put/seqid=0 2023-09-29T06:13:36,738 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/f812d3ee16bb479cb4faaf3b751a3c52 2023-09-29T06:13:36,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/e7e28f8579a842f78b33e3affd5af2c5 is 50, key is test_row_0/C:col10/1695968016063/Put/seqid=0 2023-09-29T06:13:36,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:36,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968076805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:36,806 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/e7e28f8579a842f78b33e3affd5af2c5 2023-09-29T06:13:36,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:36,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41454 deadline: 1695968076815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:36,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:36,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968076817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:36,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:36,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:36,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41426 deadline: 1695968076818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:36,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968076818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:36,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/99c7d22c33b94115a55e0f7fcd53e5fc as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/99c7d22c33b94115a55e0f7fcd53e5fc 2023-09-29T06:13:36,826 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/99c7d22c33b94115a55e0f7fcd53e5fc, entries=150, sequenceid=16, filesize=30.2 K 2023-09-29T06:13:36,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/f812d3ee16bb479cb4faaf3b751a3c52 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/f812d3ee16bb479cb4faaf3b751a3c52 2023-09-29T06:13:36,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2023-09-29T06:13:36,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:36,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. as already flushing 2023-09-29T06:13:36,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:36,835 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(53): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:36,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:36,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:36,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/f812d3ee16bb479cb4faaf3b751a3c52, entries=150, sequenceid=16, filesize=11.7 K 2023-09-29T06:13:36,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/e7e28f8579a842f78b33e3affd5af2c5 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/e7e28f8579a842f78b33e3affd5af2c5 2023-09-29T06:13:36,871 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/e7e28f8579a842f78b33e3affd5af2c5, entries=150, sequenceid=16, filesize=11.7 K 2023-09-29T06:13:36,872 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 1668fccb65cfe5ce16ea12b0cc0df5e4 in 804ms, sequenceid=16, compaction requested=false 2023-09-29T06:13:36,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 1668fccb65cfe5ce16ea12b0cc0df5e4: 2023-09-29T06:13:36,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2023-09-29T06:13:36,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:36,994 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2819): Flushing 1668fccb65cfe5ce16ea12b0cc0df5e4 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2023-09-29T06:13:36,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=A 2023-09-29T06:13:36,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:36,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=B 2023-09-29T06:13:36,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:36,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=C 2023-09-29T06:13:36,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:37,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309291cfaf382debc41d5b527ee87036fe8bb_1668fccb65cfe5ce16ea12b0cc0df5e4 is 50, key is test_row_0/A:col10/1695968016120/Put/seqid=0 2023-09-29T06:13:37,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=48 2023-09-29T06:13:37,225 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2023-09-29T06:13:37,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:37,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. as already flushing 2023-09-29T06:13:37,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:37,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:37,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41454 deadline: 1695968077325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:37,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:37,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41426 deadline: 1695968077327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:37,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:37,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968077325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:37,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968077327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:37,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:37,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968077328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:37,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:37,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968077430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:37,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:37,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41426 deadline: 1695968077430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:37,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:37,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968077430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:37,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:37,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41454 deadline: 1695968077431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:37,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:37,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968077431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:37,476 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309291cfaf382debc41d5b527ee87036fe8bb_1668fccb65cfe5ce16ea12b0cc0df5e4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309291cfaf382debc41d5b527ee87036fe8bb_1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:37,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/96698babbf764e2ea94216be33f16da5, store: [table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:13:37,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/96698babbf764e2ea94216be33f16da5 is 175, key is test_row_0/A:col10/1695968016120/Put/seqid=0 2023-09-29T06:13:37,491 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/96698babbf764e2ea94216be33f16da5 2023-09-29T06:13:37,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/b3370aaff89d428c8d24eff9bd51c8eb is 50, key is test_row_0/B:col10/1695968016120/Put/seqid=0 2023-09-29T06:13:37,533 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/b3370aaff89d428c8d24eff9bd51c8eb 2023-09-29T06:13:37,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/9380b95dfa3049dd96058607cbe0f233 is 50, key is test_row_0/C:col10/1695968016120/Put/seqid=0 2023-09-29T06:13:37,581 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/9380b95dfa3049dd96058607cbe0f233 2023-09-29T06:13:37,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/96698babbf764e2ea94216be33f16da5 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/96698babbf764e2ea94216be33f16da5 2023-09-29T06:13:37,594 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/96698babbf764e2ea94216be33f16da5, entries=150, sequenceid=41, filesize=30.2 K 2023-09-29T06:13:37,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/b3370aaff89d428c8d24eff9bd51c8eb as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/b3370aaff89d428c8d24eff9bd51c8eb 2023-09-29T06:13:37,603 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/b3370aaff89d428c8d24eff9bd51c8eb, entries=150, sequenceid=41, filesize=11.7 K 2023-09-29T06:13:37,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/9380b95dfa3049dd96058607cbe0f233 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/9380b95dfa3049dd96058607cbe0f233 2023-09-29T06:13:37,610 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/9380b95dfa3049dd96058607cbe0f233, entries=150, sequenceid=41, filesize=11.7 K 2023-09-29T06:13:37,611 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3022): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 1668fccb65cfe5ce16ea12b0cc0df5e4 in 617ms, sequenceid=41, compaction requested=false 2023-09-29T06:13:37,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2520): Flush status journal for 1668fccb65cfe5ce16ea12b0cc0df5e4: 2023-09-29T06:13:37,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:37,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=49 2023-09-29T06:13:37,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=49 2023-09-29T06:13:37,614 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=49, resume processing ppid=48 2023-09-29T06:13:37,614 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=49, ppid=48, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5740 sec 2023-09-29T06:13:37,618 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=48, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.5990 sec 2023-09-29T06:13:37,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:37,644 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 1668fccb65cfe5ce16ea12b0cc0df5e4 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2023-09-29T06:13:37,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=A 2023-09-29T06:13:37,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:37,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=B 2023-09-29T06:13:37,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:37,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=C 2023-09-29T06:13:37,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:37,671 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929ba4817f815274af6801b5fcf54311055_1668fccb65cfe5ce16ea12b0cc0df5e4 is 50, key is test_row_0/A:col10/1695968017644/Put/seqid=0 2023-09-29T06:13:37,689 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929ba4817f815274af6801b5fcf54311055_1668fccb65cfe5ce16ea12b0cc0df5e4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929ba4817f815274af6801b5fcf54311055_1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:37,691 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/fad17e62f93a488cad0e9cb91ab3f9b6, store: [table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:13:37,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/fad17e62f93a488cad0e9cb91ab3f9b6 is 175, key is test_row_0/A:col10/1695968017644/Put/seqid=0 2023-09-29T06:13:37,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:37,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968077701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:37,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:37,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41454 deadline: 1695968077701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:37,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:37,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968077704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:37,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:37,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:37,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968077705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:37,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41426 deadline: 1695968077706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:37,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:37,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968077810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:37,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:37,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41426 deadline: 1695968077811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:37,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:37,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968077811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:37,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:37,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968077814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:37,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:37,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41454 deadline: 1695968077814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:38,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:38,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41426 deadline: 1695968078014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:38,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:38,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:38,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968078014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:38,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968078014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:38,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:38,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968078017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:38,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:38,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41454 deadline: 1695968078018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:38,115 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=57, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/fad17e62f93a488cad0e9cb91ab3f9b6 2023-09-29T06:13:38,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/dc75ed81e7274a4c8fc9d30ae3c50556 is 50, key is test_row_0/B:col10/1695968017644/Put/seqid=0 2023-09-29T06:13:38,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=48 2023-09-29T06:13:38,145 INFO [Thread-842 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 48 completed 2023-09-29T06:13:38,148 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/dc75ed81e7274a4c8fc9d30ae3c50556 2023-09-29T06:13:38,148 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:13:38,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2023-09-29T06:13:38,160 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:13:38,161 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:13:38,161 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:13:38,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=50 2023-09-29T06:13:38,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/0c4f8455e7d14c1a8c2461f0f5158987 is 50, key is test_row_0/C:col10/1695968017644/Put/seqid=0 2023-09-29T06:13:38,200 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/0c4f8455e7d14c1a8c2461f0f5158987 2023-09-29T06:13:38,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/fad17e62f93a488cad0e9cb91ab3f9b6 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/fad17e62f93a488cad0e9cb91ab3f9b6 2023-09-29T06:13:38,222 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/fad17e62f93a488cad0e9cb91ab3f9b6, entries=250, sequenceid=57, filesize=47.0 K 2023-09-29T06:13:38,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/dc75ed81e7274a4c8fc9d30ae3c50556 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/dc75ed81e7274a4c8fc9d30ae3c50556 2023-09-29T06:13:38,231 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/dc75ed81e7274a4c8fc9d30ae3c50556, entries=150, sequenceid=57, filesize=11.7 K 2023-09-29T06:13:38,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/0c4f8455e7d14c1a8c2461f0f5158987 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/0c4f8455e7d14c1a8c2461f0f5158987 2023-09-29T06:13:38,245 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/0c4f8455e7d14c1a8c2461f0f5158987, entries=150, sequenceid=57, filesize=11.7 K 2023-09-29T06:13:38,248 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 1668fccb65cfe5ce16ea12b0cc0df5e4 in 604ms, sequenceid=57, compaction requested=true 2023-09-29T06:13:38,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 1668fccb65cfe5ce16ea12b0cc0df5e4: 2023-09-29T06:13:38,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1668fccb65cfe5ce16ea12b0cc0df5e4:A, priority=-2147483648, current under compaction store size is 1 2023-09-29T06:13:38,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:38,248 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:13:38,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1668fccb65cfe5ce16ea12b0cc0df5e4:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:13:38,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:13:38,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1668fccb65cfe5ce16ea12b0cc0df5e4:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:13:38,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2023-09-29T06:13:38,248 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:13:38,249 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 109998 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:13:38,250 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 1668fccb65cfe5ce16ea12b0cc0df5e4/A is initiating minor compaction (all files) 2023-09-29T06:13:38,250 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1668fccb65cfe5ce16ea12b0cc0df5e4/A in TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:38,250 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/99c7d22c33b94115a55e0f7fcd53e5fc, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/96698babbf764e2ea94216be33f16da5, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/fad17e62f93a488cad0e9cb91ab3f9b6] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp, totalSize=107.4 K 2023-09-29T06:13:38,250 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:38,250 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. files: [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/99c7d22c33b94115a55e0f7fcd53e5fc, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/96698babbf764e2ea94216be33f16da5, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/fad17e62f93a488cad0e9cb91ab3f9b6] 2023-09-29T06:13:38,251 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35952 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:13:38,251 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 99c7d22c33b94115a55e0f7fcd53e5fc, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1695968016045 2023-09-29T06:13:38,251 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 1668fccb65cfe5ce16ea12b0cc0df5e4/B is initiating minor compaction (all files) 2023-09-29T06:13:38,251 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1668fccb65cfe5ce16ea12b0cc0df5e4/B in TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:38,251 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/f812d3ee16bb479cb4faaf3b751a3c52, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/b3370aaff89d428c8d24eff9bd51c8eb, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/dc75ed81e7274a4c8fc9d30ae3c50556] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp, totalSize=35.1 K 2023-09-29T06:13:38,251 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 96698babbf764e2ea94216be33f16da5, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1695968016117 2023-09-29T06:13:38,251 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting f812d3ee16bb479cb4faaf3b751a3c52, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1695968016045 2023-09-29T06:13:38,252 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting fad17e62f93a488cad0e9cb91ab3f9b6, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1695968017326 2023-09-29T06:13:38,252 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting b3370aaff89d428c8d24eff9bd51c8eb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1695968016117 2023-09-29T06:13:38,253 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting dc75ed81e7274a4c8fc9d30ae3c50556, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1695968017326 2023-09-29T06:13:38,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=50 2023-09-29T06:13:38,265 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:13:38,273 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202309294b58258c1c6840e3ac99d201c3c84471_1668fccb65cfe5ce16ea12b0cc0df5e4 store=[table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:13:38,281 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202309294b58258c1c6840e3ac99d201c3c84471_1668fccb65cfe5ce16ea12b0cc0df5e4, store=[table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:13:38,281 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309294b58258c1c6840e3ac99d201c3c84471_1668fccb65cfe5ce16ea12b0cc0df5e4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:13:38,294 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1668fccb65cfe5ce16ea12b0cc0df5e4#B#compaction#140 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:38,294 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/a7408cfa16174a08a43d1d2856392b19 is 50, key is test_row_0/B:col10/1695968017644/Put/seqid=0 2023-09-29T06:13:38,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2023-09-29T06:13:38,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:38,319 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2819): Flushing 1668fccb65cfe5ce16ea12b0cc0df5e4 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2023-09-29T06:13:38,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=A 2023-09-29T06:13:38,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:38,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=B 2023-09-29T06:13:38,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:38,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=C 2023-09-29T06:13:38,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:38,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. as already flushing 2023-09-29T06:13:38,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:38,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:38,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:38,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41426 deadline: 1695968078337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:38,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968078336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:38,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:38,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968078341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:38,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:38,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41454 deadline: 1695968078342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:38,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:38,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968078342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:38,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929f305958024664b1fa9daecafc3ef49eb_1668fccb65cfe5ce16ea12b0cc0df5e4 is 50, key is test_row_0/A:col10/1695968017703/Put/seqid=0 2023-09-29T06:13:38,357 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/a7408cfa16174a08a43d1d2856392b19 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/a7408cfa16174a08a43d1d2856392b19 2023-09-29T06:13:38,364 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 1668fccb65cfe5ce16ea12b0cc0df5e4/B of 1668fccb65cfe5ce16ea12b0cc0df5e4 into a7408cfa16174a08a43d1d2856392b19(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:38,364 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1668fccb65cfe5ce16ea12b0cc0df5e4: 2023-09-29T06:13:38,364 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4., storeName=1668fccb65cfe5ce16ea12b0cc0df5e4/B, priority=13, startTime=1695968018248; duration=0sec 2023-09-29T06:13:38,364 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:13:38,364 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1668fccb65cfe5ce16ea12b0cc0df5e4:B 2023-09-29T06:13:38,364 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:13:38,370 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35952 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:13:38,371 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 1668fccb65cfe5ce16ea12b0cc0df5e4/C is initiating minor compaction (all files) 2023-09-29T06:13:38,371 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1668fccb65cfe5ce16ea12b0cc0df5e4/C in TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:38,371 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/e7e28f8579a842f78b33e3affd5af2c5, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/9380b95dfa3049dd96058607cbe0f233, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/0c4f8455e7d14c1a8c2461f0f5158987] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp, totalSize=35.1 K 2023-09-29T06:13:38,371 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting e7e28f8579a842f78b33e3affd5af2c5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1695968016045 2023-09-29T06:13:38,372 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 9380b95dfa3049dd96058607cbe0f233, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1695968016117 2023-09-29T06:13:38,372 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 0c4f8455e7d14c1a8c2461f0f5158987, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1695968017326 2023-09-29T06:13:38,393 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1668fccb65cfe5ce16ea12b0cc0df5e4#C#compaction#142 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:38,393 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/2709eaf23a5844b7b41dc7d9067875b5 is 50, key is test_row_0/C:col10/1695968017644/Put/seqid=0 2023-09-29T06:13:38,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:38,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968078443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:38,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:38,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41426 deadline: 1695968078444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:38,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:38,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968078448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:38,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:38,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41454 deadline: 1695968078451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:38,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:38,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968078452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:38,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=50 2023-09-29T06:13:38,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:38,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968078647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:38,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:38,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41426 deadline: 1695968078648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:38,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:38,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968078649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:38,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:38,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41454 deadline: 1695968078654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:38,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:38,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968078655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:38,736 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1668fccb65cfe5ce16ea12b0cc0df5e4#A#compaction#139 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:38,741 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/690ebdb406554486af399eb64e9e6caf is 175, key is test_row_0/A:col10/1695968017644/Put/seqid=0 2023-09-29T06:13:38,763 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/690ebdb406554486af399eb64e9e6caf as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/690ebdb406554486af399eb64e9e6caf 2023-09-29T06:13:38,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=50 2023-09-29T06:13:38,775 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929f305958024664b1fa9daecafc3ef49eb_1668fccb65cfe5ce16ea12b0cc0df5e4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929f305958024664b1fa9daecafc3ef49eb_1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:38,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/20bb16d93b454799a89ff4f79dcb4006, store: [table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:13:38,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/20bb16d93b454799a89ff4f79dcb4006 is 175, key is test_row_0/A:col10/1695968017703/Put/seqid=0 2023-09-29T06:13:38,779 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 1668fccb65cfe5ce16ea12b0cc0df5e4/A of 1668fccb65cfe5ce16ea12b0cc0df5e4 into 690ebdb406554486af399eb64e9e6caf(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:38,779 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1668fccb65cfe5ce16ea12b0cc0df5e4: 2023-09-29T06:13:38,779 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4., storeName=1668fccb65cfe5ce16ea12b0cc0df5e4/A, priority=13, startTime=1695968018248; duration=0sec 2023-09-29T06:13:38,779 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:38,779 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1668fccb65cfe5ce16ea12b0cc0df5e4:A 2023-09-29T06:13:38,791 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=77, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/20bb16d93b454799a89ff4f79dcb4006 2023-09-29T06:13:38,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/39aba1f938054a6e957bf82487842817 is 50, key is test_row_0/B:col10/1695968017703/Put/seqid=0 2023-09-29T06:13:38,811 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/2709eaf23a5844b7b41dc7d9067875b5 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/2709eaf23a5844b7b41dc7d9067875b5 2023-09-29T06:13:38,819 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 1668fccb65cfe5ce16ea12b0cc0df5e4/C of 1668fccb65cfe5ce16ea12b0cc0df5e4 into 2709eaf23a5844b7b41dc7d9067875b5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:38,819 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1668fccb65cfe5ce16ea12b0cc0df5e4: 2023-09-29T06:13:38,819 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4., storeName=1668fccb65cfe5ce16ea12b0cc0df5e4/C, priority=13, startTime=1695968018248; duration=0sec 2023-09-29T06:13:38,819 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/39aba1f938054a6e957bf82487842817 2023-09-29T06:13:38,819 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:38,819 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1668fccb65cfe5ce16ea12b0cc0df5e4:C 2023-09-29T06:13:38,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/cc0b70518ae34bddbaf5b1917b381527 is 50, key is test_row_0/C:col10/1695968017703/Put/seqid=0 2023-09-29T06:13:38,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:38,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:38,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968078951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:38,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968078952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:38,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:38,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41426 deadline: 1695968078952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:38,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:38,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41454 deadline: 1695968078958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:38,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:38,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968078962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:39,253 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/cc0b70518ae34bddbaf5b1917b381527 2023-09-29T06:13:39,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/20bb16d93b454799a89ff4f79dcb4006 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/20bb16d93b454799a89ff4f79dcb4006 2023-09-29T06:13:39,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=50 2023-09-29T06:13:39,273 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/20bb16d93b454799a89ff4f79dcb4006, entries=150, sequenceid=77, filesize=30.2 K 2023-09-29T06:13:39,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/39aba1f938054a6e957bf82487842817 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/39aba1f938054a6e957bf82487842817 2023-09-29T06:13:39,280 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/39aba1f938054a6e957bf82487842817, entries=150, sequenceid=77, filesize=11.7 K 2023-09-29T06:13:39,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/cc0b70518ae34bddbaf5b1917b381527 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/cc0b70518ae34bddbaf5b1917b381527 2023-09-29T06:13:39,292 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/cc0b70518ae34bddbaf5b1917b381527, entries=150, sequenceid=77, filesize=11.7 K 2023-09-29T06:13:39,293 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3022): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for 1668fccb65cfe5ce16ea12b0cc0df5e4 in 974ms, sequenceid=77, compaction requested=false 2023-09-29T06:13:39,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2520): Flush status journal for 1668fccb65cfe5ce16ea12b0cc0df5e4: 2023-09-29T06:13:39,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:39,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=51 2023-09-29T06:13:39,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=51 2023-09-29T06:13:39,298 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=51, resume processing ppid=50 2023-09-29T06:13:39,299 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=51, ppid=50, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1350 sec 2023-09-29T06:13:39,300 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=50, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.1510 sec 2023-09-29T06:13:39,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:39,460 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 1668fccb65cfe5ce16ea12b0cc0df5e4 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2023-09-29T06:13:39,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=A 2023-09-29T06:13:39,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:39,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=B 2023-09-29T06:13:39,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:39,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=C 2023-09-29T06:13:39,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:39,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929f4c9931d56f242a9ac21379afc9b8cec_1668fccb65cfe5ce16ea12b0cc0df5e4 is 50, key is test_row_0/A:col10/1695968019458/Put/seqid=0 2023-09-29T06:13:39,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:39,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:39,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41426 deadline: 1695968079477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:39,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968079478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:39,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:39,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41454 deadline: 1695968079480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:39,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:39,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968079482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:39,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:39,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968079482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:39,501 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929f4c9931d56f242a9ac21379afc9b8cec_1668fccb65cfe5ce16ea12b0cc0df5e4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929f4c9931d56f242a9ac21379afc9b8cec_1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:39,503 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/e4c7e2d52d1f428fadcfedd37feeea02, store: [table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:13:39,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/e4c7e2d52d1f428fadcfedd37feeea02 is 175, key is test_row_0/A:col10/1695968019458/Put/seqid=0 2023-09-29T06:13:39,512 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=100, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/e4c7e2d52d1f428fadcfedd37feeea02 2023-09-29T06:13:39,520 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/a1356accbe344dd198ead53dd1da6bed is 50, key is test_row_0/B:col10/1695968019458/Put/seqid=0 2023-09-29T06:13:39,551 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/a1356accbe344dd198ead53dd1da6bed 2023-09-29T06:13:39,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/3fad4c170ce24039a8f25a2d4e31cd37 is 50, key is test_row_0/C:col10/1695968019458/Put/seqid=0 2023-09-29T06:13:39,567 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/3fad4c170ce24039a8f25a2d4e31cd37 2023-09-29T06:13:39,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/e4c7e2d52d1f428fadcfedd37feeea02 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/e4c7e2d52d1f428fadcfedd37feeea02 2023-09-29T06:13:39,577 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/e4c7e2d52d1f428fadcfedd37feeea02, entries=150, sequenceid=100, filesize=30.2 K 2023-09-29T06:13:39,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/a1356accbe344dd198ead53dd1da6bed as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/a1356accbe344dd198ead53dd1da6bed 2023-09-29T06:13:39,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:39,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41426 deadline: 1695968079583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:39,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:39,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968079584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:39,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:39,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41454 deadline: 1695968079584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:39,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:39,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968079586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:39,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:39,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968079586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:39,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/a1356accbe344dd198ead53dd1da6bed, entries=150, sequenceid=100, filesize=11.7 K 2023-09-29T06:13:39,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/3fad4c170ce24039a8f25a2d4e31cd37 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/3fad4c170ce24039a8f25a2d4e31cd37 2023-09-29T06:13:39,608 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/3fad4c170ce24039a8f25a2d4e31cd37, entries=150, sequenceid=100, filesize=11.7 K 2023-09-29T06:13:39,609 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 1668fccb65cfe5ce16ea12b0cc0df5e4 in 150ms, sequenceid=100, compaction requested=true 2023-09-29T06:13:39,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 1668fccb65cfe5ce16ea12b0cc0df5e4: 2023-09-29T06:13:39,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1668fccb65cfe5ce16ea12b0cc0df5e4:A, priority=-2147483648, current under compaction store size is 1 2023-09-29T06:13:39,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:39,609 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:13:39,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1668fccb65cfe5ce16ea12b0cc0df5e4:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:13:39,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:39,609 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:13:39,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1668fccb65cfe5ce16ea12b0cc0df5e4:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:13:39,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:13:39,611 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92917 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:13:39,611 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 1668fccb65cfe5ce16ea12b0cc0df5e4/A is initiating minor compaction (all files) 2023-09-29T06:13:39,611 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1668fccb65cfe5ce16ea12b0cc0df5e4/A in TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:39,611 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/690ebdb406554486af399eb64e9e6caf, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/20bb16d93b454799a89ff4f79dcb4006, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/e4c7e2d52d1f428fadcfedd37feeea02] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp, totalSize=90.7 K 2023-09-29T06:13:39,611 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:39,611 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. files: [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/690ebdb406554486af399eb64e9e6caf, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/20bb16d93b454799a89ff4f79dcb4006, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/e4c7e2d52d1f428fadcfedd37feeea02] 2023-09-29T06:13:39,612 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36055 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:13:39,612 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 1668fccb65cfe5ce16ea12b0cc0df5e4/B is initiating minor compaction (all files) 2023-09-29T06:13:39,612 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1668fccb65cfe5ce16ea12b0cc0df5e4/B in TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:39,612 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/a7408cfa16174a08a43d1d2856392b19, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/39aba1f938054a6e957bf82487842817, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/a1356accbe344dd198ead53dd1da6bed] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp, totalSize=35.2 K 2023-09-29T06:13:39,613 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 690ebdb406554486af399eb64e9e6caf, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1695968017326 2023-09-29T06:13:39,618 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting a7408cfa16174a08a43d1d2856392b19, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1695968017326 2023-09-29T06:13:39,618 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 20bb16d93b454799a89ff4f79dcb4006, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1695968017696 2023-09-29T06:13:39,619 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 39aba1f938054a6e957bf82487842817, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1695968017696 2023-09-29T06:13:39,619 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting e4c7e2d52d1f428fadcfedd37feeea02, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1695968018336 2023-09-29T06:13:39,619 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting a1356accbe344dd198ead53dd1da6bed, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1695968018336 2023-09-29T06:13:39,638 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:13:39,639 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1668fccb65cfe5ce16ea12b0cc0df5e4#B#compaction#148 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:39,640 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/a2a0e20e60674e0aa39d7d8dbf9f007d is 50, key is test_row_0/B:col10/1695968019458/Put/seqid=0 2023-09-29T06:13:39,641 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2023092903aa258449454d8dac5ebf6d7d1faf3a_1668fccb65cfe5ce16ea12b0cc0df5e4 store=[table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:13:39,643 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2023092903aa258449454d8dac5ebf6d7d1faf3a_1668fccb65cfe5ce16ea12b0cc0df5e4, store=[table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:13:39,643 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023092903aa258449454d8dac5ebf6d7d1faf3a_1668fccb65cfe5ce16ea12b0cc0df5e4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:13:39,662 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/a2a0e20e60674e0aa39d7d8dbf9f007d as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/a2a0e20e60674e0aa39d7d8dbf9f007d 2023-09-29T06:13:39,668 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 1668fccb65cfe5ce16ea12b0cc0df5e4/B of 1668fccb65cfe5ce16ea12b0cc0df5e4 into a2a0e20e60674e0aa39d7d8dbf9f007d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:39,668 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1668fccb65cfe5ce16ea12b0cc0df5e4: 2023-09-29T06:13:39,668 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4., storeName=1668fccb65cfe5ce16ea12b0cc0df5e4/B, priority=13, startTime=1695968019609; duration=0sec 2023-09-29T06:13:39,668 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:13:39,668 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1668fccb65cfe5ce16ea12b0cc0df5e4:B 2023-09-29T06:13:39,668 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:13:39,670 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36055 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:13:39,670 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 1668fccb65cfe5ce16ea12b0cc0df5e4/C is initiating minor compaction (all files) 2023-09-29T06:13:39,670 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1668fccb65cfe5ce16ea12b0cc0df5e4/C in TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:39,670 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/2709eaf23a5844b7b41dc7d9067875b5, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/cc0b70518ae34bddbaf5b1917b381527, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/3fad4c170ce24039a8f25a2d4e31cd37] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp, totalSize=35.2 K 2023-09-29T06:13:39,670 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 2709eaf23a5844b7b41dc7d9067875b5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1695968017326 2023-09-29T06:13:39,671 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting cc0b70518ae34bddbaf5b1917b381527, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1695968017696 2023-09-29T06:13:39,671 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 3fad4c170ce24039a8f25a2d4e31cd37, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1695968018336 2023-09-29T06:13:39,692 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1668fccb65cfe5ce16ea12b0cc0df5e4#C#compaction#150 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:39,692 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/4e3967762ca04124bd413245751139da is 50, key is test_row_0/C:col10/1695968019458/Put/seqid=0 2023-09-29T06:13:39,715 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/4e3967762ca04124bd413245751139da as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/4e3967762ca04124bd413245751139da 2023-09-29T06:13:39,722 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 1668fccb65cfe5ce16ea12b0cc0df5e4/C of 1668fccb65cfe5ce16ea12b0cc0df5e4 into 4e3967762ca04124bd413245751139da(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:39,722 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1668fccb65cfe5ce16ea12b0cc0df5e4: 2023-09-29T06:13:39,722 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4., storeName=1668fccb65cfe5ce16ea12b0cc0df5e4/C, priority=13, startTime=1695968019609; duration=0sec 2023-09-29T06:13:39,722 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:39,722 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1668fccb65cfe5ce16ea12b0cc0df5e4:C 2023-09-29T06:13:39,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:39,792 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 1668fccb65cfe5ce16ea12b0cc0df5e4 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2023-09-29T06:13:39,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=A 2023-09-29T06:13:39,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:39,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=B 2023-09-29T06:13:39,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:39,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=C 2023-09-29T06:13:39,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:39,804 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023092913b422afda3f40bb9c27bacbf1ed15dd_1668fccb65cfe5ce16ea12b0cc0df5e4 is 50, key is test_row_0/A:col10/1695968019791/Put/seqid=0 2023-09-29T06:13:39,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:39,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:39,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968079835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:39,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968079835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:39,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:39,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968079838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:39,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:39,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41454 deadline: 1695968079839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:39,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:39,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41426 deadline: 1695968079839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:39,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:39,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:39,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968079940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:39,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968079941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:39,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:39,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968079943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:39,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:39,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41454 deadline: 1695968079944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:39,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:39,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41426 deadline: 1695968079944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:40,058 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1668fccb65cfe5ce16ea12b0cc0df5e4#A#compaction#149 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:40,058 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/40325d109222441a83e5e6f4d54aead8 is 175, key is test_row_0/A:col10/1695968019458/Put/seqid=0 2023-09-29T06:13:40,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:40,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968080143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:40,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:40,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968080144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:40,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:40,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968080147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:40,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:40,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41426 deadline: 1695968080148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:40,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:40,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41454 deadline: 1695968080148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:40,221 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023092913b422afda3f40bb9c27bacbf1ed15dd_1668fccb65cfe5ce16ea12b0cc0df5e4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023092913b422afda3f40bb9c27bacbf1ed15dd_1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:40,222 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/b1f19f332dad415687a1955cc6cfecab, store: [table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:13:40,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/b1f19f332dad415687a1955cc6cfecab is 175, key is test_row_0/A:col10/1695968019791/Put/seqid=0 2023-09-29T06:13:40,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=50 2023-09-29T06:13:40,272 INFO [Thread-842 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 50 completed 2023-09-29T06:13:40,273 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:13:40,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2023-09-29T06:13:40,277 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:13:40,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=52 2023-09-29T06:13:40,278 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:13:40,278 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:13:40,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=52 2023-09-29T06:13:40,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2023-09-29T06:13:40,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:40,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. as already flushing 2023-09-29T06:13:40,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:40,432 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(53): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:40,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:40,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:40,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:40,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968080446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:40,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:40,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968080447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:40,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:40,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968080450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:40,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:40,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41426 deadline: 1695968080453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:40,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:40,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41454 deadline: 1695968080454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:40,476 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/40325d109222441a83e5e6f4d54aead8 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/40325d109222441a83e5e6f4d54aead8 2023-09-29T06:13:40,485 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 1668fccb65cfe5ce16ea12b0cc0df5e4/A of 1668fccb65cfe5ce16ea12b0cc0df5e4 into 40325d109222441a83e5e6f4d54aead8(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:40,485 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1668fccb65cfe5ce16ea12b0cc0df5e4: 2023-09-29T06:13:40,485 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4., storeName=1668fccb65cfe5ce16ea12b0cc0df5e4/A, priority=13, startTime=1695968019609; duration=0sec 2023-09-29T06:13:40,485 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:40,485 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1668fccb65cfe5ce16ea12b0cc0df5e4:A 2023-09-29T06:13:40,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=52 2023-09-29T06:13:40,583 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2023-09-29T06:13:40,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:40,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. as already flushing 2023-09-29T06:13:40,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:40,584 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(53): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:40,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:40,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:40,635 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=121, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/b1f19f332dad415687a1955cc6cfecab 2023-09-29T06:13:40,651 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/7b23583faefa45929d0e787c3296c92e is 50, key is test_row_0/B:col10/1695968019791/Put/seqid=0 2023-09-29T06:13:40,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2023-09-29T06:13:40,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:40,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. as already flushing 2023-09-29T06:13:40,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:40,739 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(53): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:40,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:40,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:40,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=52 2023-09-29T06:13:40,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2023-09-29T06:13:40,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:40,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. as already flushing 2023-09-29T06:13:40,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:40,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(53): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:40,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:40,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:40,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:40,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968080952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:40,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:40,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41426 deadline: 1695968080954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:40,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:40,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968080955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:40,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:40,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41454 deadline: 1695968080957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:40,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:40,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968080959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:41,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2023-09-29T06:13:41,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:41,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. as already flushing 2023-09-29T06:13:41,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:41,046 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(53): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:41,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:41,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:41,062 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/7b23583faefa45929d0e787c3296c92e 2023-09-29T06:13:41,074 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/1b3e450cfdbe4ca1917600b4e2cc6639 is 50, key is test_row_0/C:col10/1695968019791/Put/seqid=0 2023-09-29T06:13:41,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2023-09-29T06:13:41,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:41,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. as already flushing 2023-09-29T06:13:41,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:41,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(53): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:41,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:41,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:41,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2023-09-29T06:13:41,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:41,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. as already flushing 2023-09-29T06:13:41,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:41,356 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(53): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:41,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:41,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:41,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=52 2023-09-29T06:13:41,486 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/1b3e450cfdbe4ca1917600b4e2cc6639 2023-09-29T06:13:41,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/b1f19f332dad415687a1955cc6cfecab as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/b1f19f332dad415687a1955cc6cfecab 2023-09-29T06:13:41,498 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/b1f19f332dad415687a1955cc6cfecab, entries=150, sequenceid=121, filesize=30.2 K 2023-09-29T06:13:41,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/7b23583faefa45929d0e787c3296c92e as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/7b23583faefa45929d0e787c3296c92e 2023-09-29T06:13:41,504 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/7b23583faefa45929d0e787c3296c92e, entries=150, sequenceid=121, filesize=11.7 K 2023-09-29T06:13:41,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/1b3e450cfdbe4ca1917600b4e2cc6639 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/1b3e450cfdbe4ca1917600b4e2cc6639 2023-09-29T06:13:41,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2023-09-29T06:13:41,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:41,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. as already flushing 2023-09-29T06:13:41,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:41,510 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(53): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:41,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:41,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:41,512 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/1b3e450cfdbe4ca1917600b4e2cc6639, entries=150, sequenceid=121, filesize=11.7 K 2023-09-29T06:13:41,513 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for 1668fccb65cfe5ce16ea12b0cc0df5e4 in 1721ms, sequenceid=121, compaction requested=false 2023-09-29T06:13:41,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 1668fccb65cfe5ce16ea12b0cc0df5e4: 2023-09-29T06:13:41,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2023-09-29T06:13:41,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:41,663 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2819): Flushing 1668fccb65cfe5ce16ea12b0cc0df5e4 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2023-09-29T06:13:41,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=A 2023-09-29T06:13:41,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:41,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=B 2023-09-29T06:13:41,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:41,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=C 2023-09-29T06:13:41,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:41,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309296041c21273f24023bf13f8016c83e89c_1668fccb65cfe5ce16ea12b0cc0df5e4 is 50, key is test_row_0/A:col10/1695968019797/Put/seqid=0 2023-09-29T06:13:41,731 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309296041c21273f24023bf13f8016c83e89c_1668fccb65cfe5ce16ea12b0cc0df5e4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309296041c21273f24023bf13f8016c83e89c_1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:41,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/6ee86473ab2343a788657ec685706263, store: [table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:13:41,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/6ee86473ab2343a788657ec685706263 is 175, key is test_row_0/A:col10/1695968019797/Put/seqid=0 2023-09-29T06:13:41,776 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=140, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/6ee86473ab2343a788657ec685706263 2023-09-29T06:13:41,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/e9bd2b6eb08e4e62bc101f48c1636b8a is 50, key is test_row_0/B:col10/1695968019797/Put/seqid=0 2023-09-29T06:13:41,815 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/e9bd2b6eb08e4e62bc101f48c1636b8a 2023-09-29T06:13:41,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/c193c67c3e61499ba003bfc55f347311 is 50, key is test_row_0/C:col10/1695968019797/Put/seqid=0 2023-09-29T06:13:41,833 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/c193c67c3e61499ba003bfc55f347311 2023-09-29T06:13:41,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/6ee86473ab2343a788657ec685706263 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/6ee86473ab2343a788657ec685706263 2023-09-29T06:13:41,848 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/6ee86473ab2343a788657ec685706263, entries=150, sequenceid=140, filesize=30.3 K 2023-09-29T06:13:41,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/e9bd2b6eb08e4e62bc101f48c1636b8a as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/e9bd2b6eb08e4e62bc101f48c1636b8a 2023-09-29T06:13:41,856 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/e9bd2b6eb08e4e62bc101f48c1636b8a, entries=150, sequenceid=140, filesize=11.8 K 2023-09-29T06:13:41,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/c193c67c3e61499ba003bfc55f347311 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/c193c67c3e61499ba003bfc55f347311 2023-09-29T06:13:41,880 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/c193c67c3e61499ba003bfc55f347311, entries=150, sequenceid=140, filesize=11.8 K 2023-09-29T06:13:41,881 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3022): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=0 B/0 for 1668fccb65cfe5ce16ea12b0cc0df5e4 in 218ms, sequenceid=140, compaction requested=true 2023-09-29T06:13:41,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2520): Flush status journal for 1668fccb65cfe5ce16ea12b0cc0df5e4: 2023-09-29T06:13:41,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:41,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=53 2023-09-29T06:13:41,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=53 2023-09-29T06:13:41,884 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=53, resume processing ppid=52 2023-09-29T06:13:41,885 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=53, ppid=52, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6050 sec 2023-09-29T06:13:41,887 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=52, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.6120 sec 2023-09-29T06:13:42,004 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 1668fccb65cfe5ce16ea12b0cc0df5e4 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2023-09-29T06:13:42,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=A 2023-09-29T06:13:42,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:42,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=B 2023-09-29T06:13:42,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:42,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=C 2023-09-29T06:13:42,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:42,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:42,030 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023092994285a1b0ddc4cfaa8ecb5ac986ee07f_1668fccb65cfe5ce16ea12b0cc0df5e4 is 50, key is test_row_0/A:col10/1695968022000/Put/seqid=0 2023-09-29T06:13:42,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:42,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968082053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:42,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:42,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41454 deadline: 1695968082055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:42,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:42,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41426 deadline: 1695968082056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:42,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:42,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968082057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:42,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:42,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968082059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:42,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:42,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41454 deadline: 1695968082167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:42,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:42,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968082167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:42,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:42,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41426 deadline: 1695968082168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:42,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:42,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968082168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:42,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:42,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968082168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:42,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:42,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41454 deadline: 1695968082369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:42,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:42,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41426 deadline: 1695968082370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:42,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:42,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968082370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:42,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:42,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968082371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:42,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:42,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968082372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:42,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=52 2023-09-29T06:13:42,387 INFO [Thread-842 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 52 completed 2023-09-29T06:13:42,389 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:13:42,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2023-09-29T06:13:42,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=54 2023-09-29T06:13:42,394 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:13:42,394 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:13:42,394 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:13:42,464 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023092994285a1b0ddc4cfaa8ecb5ac986ee07f_1668fccb65cfe5ce16ea12b0cc0df5e4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023092994285a1b0ddc4cfaa8ecb5ac986ee07f_1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:42,466 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/ca9d61755642409ca623a485af340e44, store: [table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:13:42,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/ca9d61755642409ca623a485af340e44 is 175, key is test_row_0/A:col10/1695968022000/Put/seqid=0 2023-09-29T06:13:42,474 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=151, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/ca9d61755642409ca623a485af340e44 2023-09-29T06:13:42,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/4b4ce7d6d60b46b99698f597cb331aad is 50, key is test_row_0/B:col10/1695968022000/Put/seqid=0 2023-09-29T06:13:42,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=54 2023-09-29T06:13:42,496 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/4b4ce7d6d60b46b99698f597cb331aad 2023-09-29T06:13:42,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/340cdfa54bd24ae6a863c7546b841b97 is 50, key is test_row_0/C:col10/1695968022000/Put/seqid=0 2023-09-29T06:13:42,540 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/340cdfa54bd24ae6a863c7546b841b97 2023-09-29T06:13:42,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/ca9d61755642409ca623a485af340e44 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/ca9d61755642409ca623a485af340e44 2023-09-29T06:13:42,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2023-09-29T06:13:42,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:42,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. as already flushing 2023-09-29T06:13:42,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:42,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(53): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:42,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:42,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:42,555 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/ca9d61755642409ca623a485af340e44, entries=400, sequenceid=151, filesize=72.6 K 2023-09-29T06:13:42,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/4b4ce7d6d60b46b99698f597cb331aad as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/4b4ce7d6d60b46b99698f597cb331aad 2023-09-29T06:13:42,562 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/4b4ce7d6d60b46b99698f597cb331aad, entries=150, sequenceid=151, filesize=11.8 K 2023-09-29T06:13:42,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/340cdfa54bd24ae6a863c7546b841b97 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/340cdfa54bd24ae6a863c7546b841b97 2023-09-29T06:13:42,572 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2023-09-29T06:13:42,572 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2023-09-29T06:13:42,573 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/340cdfa54bd24ae6a863c7546b841b97, entries=150, sequenceid=151, filesize=11.8 K 2023-09-29T06:13:42,582 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 1668fccb65cfe5ce16ea12b0cc0df5e4 in 578ms, sequenceid=151, compaction requested=true 2023-09-29T06:13:42,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 1668fccb65cfe5ce16ea12b0cc0df5e4: 2023-09-29T06:13:42,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1668fccb65cfe5ce16ea12b0cc0df5e4:A, priority=-2147483648, current under compaction store size is 1 2023-09-29T06:13:42,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:42,583 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-29T06:13:42,583 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-29T06:13:42,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1668fccb65cfe5ce16ea12b0cc0df5e4:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:13:42,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:42,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1668fccb65cfe5ce16ea12b0cc0df5e4:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:13:42,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:13:42,585 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48392 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-29T06:13:42,585 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 1668fccb65cfe5ce16ea12b0cc0df5e4/B is initiating minor compaction (all files) 2023-09-29T06:13:42,585 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1668fccb65cfe5ce16ea12b0cc0df5e4/B in TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:42,586 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/a2a0e20e60674e0aa39d7d8dbf9f007d, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/7b23583faefa45929d0e787c3296c92e, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/e9bd2b6eb08e4e62bc101f48c1636b8a, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/4b4ce7d6d60b46b99698f597cb331aad] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp, totalSize=47.3 K 2023-09-29T06:13:42,586 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 167498 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-29T06:13:42,586 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 1668fccb65cfe5ce16ea12b0cc0df5e4/A is initiating minor compaction (all files) 2023-09-29T06:13:42,586 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1668fccb65cfe5ce16ea12b0cc0df5e4/A in TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:42,586 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/40325d109222441a83e5e6f4d54aead8, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/b1f19f332dad415687a1955cc6cfecab, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/6ee86473ab2343a788657ec685706263, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/ca9d61755642409ca623a485af340e44] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp, totalSize=163.6 K 2023-09-29T06:13:42,586 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:42,586 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. files: [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/40325d109222441a83e5e6f4d54aead8, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/b1f19f332dad415687a1955cc6cfecab, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/6ee86473ab2343a788657ec685706263, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/ca9d61755642409ca623a485af340e44] 2023-09-29T06:13:42,587 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting a2a0e20e60674e0aa39d7d8dbf9f007d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1695968018336 2023-09-29T06:13:42,587 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 7b23583faefa45929d0e787c3296c92e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1695968019472 2023-09-29T06:13:42,588 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting e9bd2b6eb08e4e62bc101f48c1636b8a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1695968019797 2023-09-29T06:13:42,588 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 40325d109222441a83e5e6f4d54aead8, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1695968018336 2023-09-29T06:13:42,588 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 4b4ce7d6d60b46b99698f597cb331aad, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1695968022000 2023-09-29T06:13:42,591 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting b1f19f332dad415687a1955cc6cfecab, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1695968019472 2023-09-29T06:13:42,592 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 6ee86473ab2343a788657ec685706263, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1695968019797 2023-09-29T06:13:42,594 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting ca9d61755642409ca623a485af340e44, keycount=400, bloomtype=ROW, size=72.6 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1695968021988 2023-09-29T06:13:42,602 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1668fccb65cfe5ce16ea12b0cc0df5e4#B#compaction#160 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:42,603 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/8d0b29d3d0b143aaa452583c001ce668 is 50, key is test_row_0/B:col10/1695968022000/Put/seqid=0 2023-09-29T06:13:42,605 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:13:42,611 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20230929f52e720c5f324e0fac2e31eb067e71d0_1668fccb65cfe5ce16ea12b0cc0df5e4 store=[table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:13:42,615 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20230929f52e720c5f324e0fac2e31eb067e71d0_1668fccb65cfe5ce16ea12b0cc0df5e4, store=[table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:13:42,615 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929f52e720c5f324e0fac2e31eb067e71d0_1668fccb65cfe5ce16ea12b0cc0df5e4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:13:42,643 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/8d0b29d3d0b143aaa452583c001ce668 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/8d0b29d3d0b143aaa452583c001ce668 2023-09-29T06:13:42,650 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 1668fccb65cfe5ce16ea12b0cc0df5e4/B of 1668fccb65cfe5ce16ea12b0cc0df5e4 into 8d0b29d3d0b143aaa452583c001ce668(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:42,650 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1668fccb65cfe5ce16ea12b0cc0df5e4: 2023-09-29T06:13:42,650 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4., storeName=1668fccb65cfe5ce16ea12b0cc0df5e4/B, priority=12, startTime=1695968022583; duration=0sec 2023-09-29T06:13:42,650 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:13:42,651 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1668fccb65cfe5ce16ea12b0cc0df5e4:B 2023-09-29T06:13:42,651 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-29T06:13:42,653 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48392 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-29T06:13:42,653 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 1668fccb65cfe5ce16ea12b0cc0df5e4/C is initiating minor compaction (all files) 2023-09-29T06:13:42,653 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1668fccb65cfe5ce16ea12b0cc0df5e4/C in TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:42,653 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/4e3967762ca04124bd413245751139da, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/1b3e450cfdbe4ca1917600b4e2cc6639, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/c193c67c3e61499ba003bfc55f347311, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/340cdfa54bd24ae6a863c7546b841b97] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp, totalSize=47.3 K 2023-09-29T06:13:42,653 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1668fccb65cfe5ce16ea12b0cc0df5e4#A#compaction#161 average throughput is 0.51 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:42,654 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 4e3967762ca04124bd413245751139da, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1695968018336 2023-09-29T06:13:42,654 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/9d1fb5935ab9487dbbb5940409e5be07 is 175, key is test_row_0/A:col10/1695968022000/Put/seqid=0 2023-09-29T06:13:42,654 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 1b3e450cfdbe4ca1917600b4e2cc6639, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1695968019472 2023-09-29T06:13:42,655 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting c193c67c3e61499ba003bfc55f347311, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1695968019797 2023-09-29T06:13:42,656 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 340cdfa54bd24ae6a863c7546b841b97, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1695968022000 2023-09-29T06:13:42,669 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1668fccb65cfe5ce16ea12b0cc0df5e4#C#compaction#162 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:42,670 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/7bb1145b4be749f4a9acac33c1bf4d38 is 50, key is test_row_0/C:col10/1695968022000/Put/seqid=0 2023-09-29T06:13:42,676 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 1668fccb65cfe5ce16ea12b0cc0df5e4 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2023-09-29T06:13:42,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:42,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=A 2023-09-29T06:13:42,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:42,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=B 2023-09-29T06:13:42,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:42,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=C 2023-09-29T06:13:42,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:42,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309294bdd738d8853443b86eb1dbcb5a1886a_1668fccb65cfe5ce16ea12b0cc0df5e4 is 50, key is test_row_0/A:col10/1695968022673/Put/seqid=0 2023-09-29T06:13:42,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=54 2023-09-29T06:13:42,700 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309294bdd738d8853443b86eb1dbcb5a1886a_1668fccb65cfe5ce16ea12b0cc0df5e4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309294bdd738d8853443b86eb1dbcb5a1886a_1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:42,701 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/ce362edc2c30416abd3fcc2a8538dce4, store: [table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:13:42,701 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/ce362edc2c30416abd3fcc2a8538dce4 is 175, key is test_row_0/A:col10/1695968022673/Put/seqid=0 2023-09-29T06:13:42,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2023-09-29T06:13:42,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:42,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. as already flushing 2023-09-29T06:13:42,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:42,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(53): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:42,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:42,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:42,708 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=178, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/ce362edc2c30416abd3fcc2a8538dce4 2023-09-29T06:13:42,723 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/2890bf40263b4b7e86e9bb811e28ff4b is 50, key is test_row_0/B:col10/1695968022673/Put/seqid=0 2023-09-29T06:13:42,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:42,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41454 deadline: 1695968082713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:42,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:42,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968082720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:42,730 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/2890bf40263b4b7e86e9bb811e28ff4b 2023-09-29T06:13:42,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:42,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968082727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:42,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:42,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968082730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:42,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:42,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41426 deadline: 1695968082739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:42,751 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/2fa8a2ab04d54a968000addaff7025e6 is 50, key is test_row_0/C:col10/1695968022673/Put/seqid=0 2023-09-29T06:13:42,762 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/2fa8a2ab04d54a968000addaff7025e6 2023-09-29T06:13:42,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/ce362edc2c30416abd3fcc2a8538dce4 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/ce362edc2c30416abd3fcc2a8538dce4 2023-09-29T06:13:42,773 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/ce362edc2c30416abd3fcc2a8538dce4, entries=200, sequenceid=178, filesize=38.8 K 2023-09-29T06:13:42,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/2890bf40263b4b7e86e9bb811e28ff4b as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/2890bf40263b4b7e86e9bb811e28ff4b 2023-09-29T06:13:42,779 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/2890bf40263b4b7e86e9bb811e28ff4b, entries=150, sequenceid=178, filesize=11.8 K 2023-09-29T06:13:42,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/2fa8a2ab04d54a968000addaff7025e6 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/2fa8a2ab04d54a968000addaff7025e6 2023-09-29T06:13:42,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/2fa8a2ab04d54a968000addaff7025e6, entries=150, sequenceid=178, filesize=11.8 K 2023-09-29T06:13:42,788 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 1668fccb65cfe5ce16ea12b0cc0df5e4 in 113ms, sequenceid=178, compaction requested=false 2023-09-29T06:13:42,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 1668fccb65cfe5ce16ea12b0cc0df5e4: 2023-09-29T06:13:42,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:42,831 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 1668fccb65cfe5ce16ea12b0cc0df5e4 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2023-09-29T06:13:42,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=A 2023-09-29T06:13:42,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:42,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=B 2023-09-29T06:13:42,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:42,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=C 2023-09-29T06:13:42,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:42,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309296facf53b66c54cbab05b2bf75daf4abb_1668fccb65cfe5ce16ea12b0cc0df5e4 is 50, key is test_row_0/A:col10/1695968022712/Put/seqid=0 2023-09-29T06:13:42,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2023-09-29T06:13:42,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:42,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. as already flushing 2023-09-29T06:13:42,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:42,861 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(53): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:42,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:42,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:42,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:42,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968082858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:42,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:42,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41426 deadline: 1695968082859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:42,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:42,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968082860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:42,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:42,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968082861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:42,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:42,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41454 deadline: 1695968082874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:42,894 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309296facf53b66c54cbab05b2bf75daf4abb_1668fccb65cfe5ce16ea12b0cc0df5e4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309296facf53b66c54cbab05b2bf75daf4abb_1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:42,895 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/02d8dbe4b2dd479695f2e5afc576b206, store: [table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:13:42,896 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/02d8dbe4b2dd479695f2e5afc576b206 is 175, key is test_row_0/A:col10/1695968022712/Put/seqid=0 2023-09-29T06:13:42,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:42,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41426 deadline: 1695968082966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:42,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:42,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968082969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:42,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:42,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968082969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:42,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:42,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968082975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:42,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:42,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41454 deadline: 1695968082979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:42,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=54 2023-09-29T06:13:43,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2023-09-29T06:13:43,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:43,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. as already flushing 2023-09-29T06:13:43,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:43,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(53): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:43,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:43,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:43,091 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/9d1fb5935ab9487dbbb5940409e5be07 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/9d1fb5935ab9487dbbb5940409e5be07 2023-09-29T06:13:43,093 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/7bb1145b4be749f4a9acac33c1bf4d38 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/7bb1145b4be749f4a9acac33c1bf4d38 2023-09-29T06:13:43,097 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 1668fccb65cfe5ce16ea12b0cc0df5e4/A of 1668fccb65cfe5ce16ea12b0cc0df5e4 into 9d1fb5935ab9487dbbb5940409e5be07(size=30.7 K), total size for store is 69.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:43,097 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1668fccb65cfe5ce16ea12b0cc0df5e4: 2023-09-29T06:13:43,097 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4., storeName=1668fccb65cfe5ce16ea12b0cc0df5e4/A, priority=12, startTime=1695968022583; duration=0sec 2023-09-29T06:13:43,097 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:43,097 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1668fccb65cfe5ce16ea12b0cc0df5e4:A 2023-09-29T06:13:43,100 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 1668fccb65cfe5ce16ea12b0cc0df5e4/C of 1668fccb65cfe5ce16ea12b0cc0df5e4 into 7bb1145b4be749f4a9acac33c1bf4d38(size=12.2 K), total size for store is 24.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:43,100 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1668fccb65cfe5ce16ea12b0cc0df5e4: 2023-09-29T06:13:43,100 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4., storeName=1668fccb65cfe5ce16ea12b0cc0df5e4/C, priority=12, startTime=1695968022583; duration=0sec 2023-09-29T06:13:43,100 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:43,100 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1668fccb65cfe5ce16ea12b0cc0df5e4:C 2023-09-29T06:13:43,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:43,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41426 deadline: 1695968083171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:43,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:43,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968083176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:43,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:43,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968083177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:43,179 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2023-09-29T06:13:43,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:43,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. as already flushing 2023-09-29T06:13:43,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:43,179 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(53): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:43,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:43,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:43,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968083178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:43,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:43,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:43,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41454 deadline: 1695968083183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:43,309 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=190, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/02d8dbe4b2dd479695f2e5afc576b206 2023-09-29T06:13:43,319 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/6712827553864291b6437d5cabffb9d1 is 50, key is test_row_0/B:col10/1695968022712/Put/seqid=0 2023-09-29T06:13:43,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2023-09-29T06:13:43,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:43,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. as already flushing 2023-09-29T06:13:43,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:43,334 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(53): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:43,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:43,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:43,354 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/6712827553864291b6437d5cabffb9d1 2023-09-29T06:13:43,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/2071f61eb8ec4752a39f011ce534f4b0 is 50, key is test_row_0/C:col10/1695968022712/Put/seqid=0 2023-09-29T06:13:43,395 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/2071f61eb8ec4752a39f011ce534f4b0 2023-09-29T06:13:43,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/02d8dbe4b2dd479695f2e5afc576b206 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/02d8dbe4b2dd479695f2e5afc576b206 2023-09-29T06:13:43,408 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/02d8dbe4b2dd479695f2e5afc576b206, entries=150, sequenceid=190, filesize=30.4 K 2023-09-29T06:13:43,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/6712827553864291b6437d5cabffb9d1 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/6712827553864291b6437d5cabffb9d1 2023-09-29T06:13:43,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/6712827553864291b6437d5cabffb9d1, entries=150, sequenceid=190, filesize=11.8 K 2023-09-29T06:13:43,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/2071f61eb8ec4752a39f011ce534f4b0 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/2071f61eb8ec4752a39f011ce534f4b0 2023-09-29T06:13:43,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/2071f61eb8ec4752a39f011ce534f4b0, entries=150, sequenceid=190, filesize=11.8 K 2023-09-29T06:13:43,422 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 1668fccb65cfe5ce16ea12b0cc0df5e4 in 591ms, sequenceid=190, compaction requested=true 2023-09-29T06:13:43,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 1668fccb65cfe5ce16ea12b0cc0df5e4: 2023-09-29T06:13:43,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1668fccb65cfe5ce16ea12b0cc0df5e4:A, priority=-2147483648, current under compaction store size is 1 2023-09-29T06:13:43,423 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:13:43,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:43,423 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:13:43,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1668fccb65cfe5ce16ea12b0cc0df5e4:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:13:43,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:43,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1668fccb65cfe5ce16ea12b0cc0df5e4:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:13:43,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:13:43,424 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102250 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:13:43,424 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 1668fccb65cfe5ce16ea12b0cc0df5e4/A is initiating minor compaction (all files) 2023-09-29T06:13:43,424 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1668fccb65cfe5ce16ea12b0cc0df5e4/A in TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:43,424 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36744 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:13:43,424 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/9d1fb5935ab9487dbbb5940409e5be07, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/ce362edc2c30416abd3fcc2a8538dce4, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/02d8dbe4b2dd479695f2e5afc576b206] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp, totalSize=99.9 K 2023-09-29T06:13:43,424 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:43,424 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 1668fccb65cfe5ce16ea12b0cc0df5e4/B is initiating minor compaction (all files) 2023-09-29T06:13:43,424 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. files: [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/9d1fb5935ab9487dbbb5940409e5be07, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/ce362edc2c30416abd3fcc2a8538dce4, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/02d8dbe4b2dd479695f2e5afc576b206] 2023-09-29T06:13:43,424 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1668fccb65cfe5ce16ea12b0cc0df5e4/B in TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:43,424 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/8d0b29d3d0b143aaa452583c001ce668, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/2890bf40263b4b7e86e9bb811e28ff4b, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/6712827553864291b6437d5cabffb9d1] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp, totalSize=35.9 K 2023-09-29T06:13:43,425 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 9d1fb5935ab9487dbbb5940409e5be07, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1695968022000 2023-09-29T06:13:43,425 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 8d0b29d3d0b143aaa452583c001ce668, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1695968022000 2023-09-29T06:13:43,425 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting ce362edc2c30416abd3fcc2a8538dce4, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1695968022053 2023-09-29T06:13:43,426 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 02d8dbe4b2dd479695f2e5afc576b206, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1695968022712 2023-09-29T06:13:43,426 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 2890bf40263b4b7e86e9bb811e28ff4b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1695968022054 2023-09-29T06:13:43,426 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 6712827553864291b6437d5cabffb9d1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1695968022712 2023-09-29T06:13:43,434 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:13:43,436 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1668fccb65cfe5ce16ea12b0cc0df5e4#B#compaction#170 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:43,437 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/b695634ad3fb4642ab75deaf324386c8 is 50, key is test_row_0/B:col10/1695968022712/Put/seqid=0 2023-09-29T06:13:43,442 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20230929fe4e93c5adc14caf8299dba386717a22_1668fccb65cfe5ce16ea12b0cc0df5e4 store=[table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:13:43,444 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20230929fe4e93c5adc14caf8299dba386717a22_1668fccb65cfe5ce16ea12b0cc0df5e4, store=[table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:13:43,444 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929fe4e93c5adc14caf8299dba386717a22_1668fccb65cfe5ce16ea12b0cc0df5e4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:13:43,473 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/b695634ad3fb4642ab75deaf324386c8 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/b695634ad3fb4642ab75deaf324386c8 2023-09-29T06:13:43,474 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1668fccb65cfe5ce16ea12b0cc0df5e4#A#compaction#169 average throughput is 0.61 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:43,475 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/5518b8580fb84fac9a6870dc6516b3e4 is 175, key is test_row_0/A:col10/1695968022712/Put/seqid=0 2023-09-29T06:13:43,479 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 1668fccb65cfe5ce16ea12b0cc0df5e4 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2023-09-29T06:13:43,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=A 2023-09-29T06:13:43,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:43,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=B 2023-09-29T06:13:43,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:43,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=C 2023-09-29T06:13:43,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:43,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:43,484 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 1668fccb65cfe5ce16ea12b0cc0df5e4/B of 1668fccb65cfe5ce16ea12b0cc0df5e4 into b695634ad3fb4642ab75deaf324386c8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:43,484 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1668fccb65cfe5ce16ea12b0cc0df5e4: 2023-09-29T06:13:43,484 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4., storeName=1668fccb65cfe5ce16ea12b0cc0df5e4/B, priority=13, startTime=1695968023423; duration=0sec 2023-09-29T06:13:43,485 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:13:43,485 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1668fccb65cfe5ce16ea12b0cc0df5e4:B 2023-09-29T06:13:43,485 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:13:43,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2023-09-29T06:13:43,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:43,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. as already flushing 2023-09-29T06:13:43,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:43,487 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(53): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:43,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:43,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:43,489 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36744 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:13:43,489 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 1668fccb65cfe5ce16ea12b0cc0df5e4/C is initiating minor compaction (all files) 2023-09-29T06:13:43,489 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1668fccb65cfe5ce16ea12b0cc0df5e4/C in TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:43,489 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/7bb1145b4be749f4a9acac33c1bf4d38, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/2fa8a2ab04d54a968000addaff7025e6, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/2071f61eb8ec4752a39f011ce534f4b0] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp, totalSize=35.9 K 2023-09-29T06:13:43,490 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 7bb1145b4be749f4a9acac33c1bf4d38, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1695968022000 2023-09-29T06:13:43,491 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 2fa8a2ab04d54a968000addaff7025e6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1695968022054 2023-09-29T06:13:43,491 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 2071f61eb8ec4752a39f011ce534f4b0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1695968022712 2023-09-29T06:13:43,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=54 2023-09-29T06:13:43,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:43,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968083490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:43,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:43,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968083496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:43,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:43,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968083496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:43,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:43,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41426 deadline: 1695968083498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:43,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:43,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41454 deadline: 1695968083499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:43,505 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1668fccb65cfe5ce16ea12b0cc0df5e4#C#compaction#171 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:43,506 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/2c0cf557040b4fa5911c8bc1f640ec85 is 50, key is test_row_0/C:col10/1695968022712/Put/seqid=0 2023-09-29T06:13:43,516 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929d4d575eb6a20451fa37ac172e1e26ad6_1668fccb65cfe5ce16ea12b0cc0df5e4 is 50, key is test_row_0/A:col10/1695968022860/Put/seqid=0 2023-09-29T06:13:43,530 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/5518b8580fb84fac9a6870dc6516b3e4 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/5518b8580fb84fac9a6870dc6516b3e4 2023-09-29T06:13:43,536 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/2c0cf557040b4fa5911c8bc1f640ec85 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/2c0cf557040b4fa5911c8bc1f640ec85 2023-09-29T06:13:43,537 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 1668fccb65cfe5ce16ea12b0cc0df5e4/A of 1668fccb65cfe5ce16ea12b0cc0df5e4 into 5518b8580fb84fac9a6870dc6516b3e4(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:43,537 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1668fccb65cfe5ce16ea12b0cc0df5e4: 2023-09-29T06:13:43,537 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4., storeName=1668fccb65cfe5ce16ea12b0cc0df5e4/A, priority=13, startTime=1695968023422; duration=0sec 2023-09-29T06:13:43,537 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:43,537 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1668fccb65cfe5ce16ea12b0cc0df5e4:A 2023-09-29T06:13:43,543 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 1668fccb65cfe5ce16ea12b0cc0df5e4/C of 1668fccb65cfe5ce16ea12b0cc0df5e4 into 2c0cf557040b4fa5911c8bc1f640ec85(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:43,543 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1668fccb65cfe5ce16ea12b0cc0df5e4: 2023-09-29T06:13:43,543 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4., storeName=1668fccb65cfe5ce16ea12b0cc0df5e4/C, priority=13, startTime=1695968023423; duration=0sec 2023-09-29T06:13:43,543 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:43,543 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1668fccb65cfe5ce16ea12b0cc0df5e4:C 2023-09-29T06:13:43,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:43,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968083602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:43,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:43,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968083603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:43,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:43,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41426 deadline: 1695968083604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:43,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:43,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41454 deadline: 1695968083605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:43,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:43,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968083606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:43,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2023-09-29T06:13:43,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:43,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. as already flushing 2023-09-29T06:13:43,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:43,640 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(53): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:43,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:43,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:43,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2023-09-29T06:13:43,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:43,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. as already flushing 2023-09-29T06:13:43,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:43,796 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(53): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:43,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:43,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:43,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:43,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968083805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:43,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:43,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41426 deadline: 1695968083806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:43,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:43,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968083806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:43,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:43,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41454 deadline: 1695968083809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:43,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:43,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968083811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:43,936 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929d4d575eb6a20451fa37ac172e1e26ad6_1668fccb65cfe5ce16ea12b0cc0df5e4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929d4d575eb6a20451fa37ac172e1e26ad6_1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:43,938 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/c3e7fbfdb7744f688b3df03af100bb90, store: [table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:13:43,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/c3e7fbfdb7744f688b3df03af100bb90 is 175, key is test_row_0/A:col10/1695968022860/Put/seqid=0 2023-09-29T06:13:43,942 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=218, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/c3e7fbfdb7744f688b3df03af100bb90 2023-09-29T06:13:43,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2023-09-29T06:13:43,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:43,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. as already flushing 2023-09-29T06:13:43,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:43,952 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(53): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:43,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:43,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:43,955 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/3e0e2153753440468db40372d268bdee is 50, key is test_row_0/B:col10/1695968022860/Put/seqid=0 2023-09-29T06:13:43,984 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/3e0e2153753440468db40372d268bdee 2023-09-29T06:13:43,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/85a33cebb05045c2858afc4be5c4ff81 is 50, key is test_row_0/C:col10/1695968022860/Put/seqid=0 2023-09-29T06:13:44,023 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/85a33cebb05045c2858afc4be5c4ff81 2023-09-29T06:13:44,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/c3e7fbfdb7744f688b3df03af100bb90 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/c3e7fbfdb7744f688b3df03af100bb90 2023-09-29T06:13:44,034 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/c3e7fbfdb7744f688b3df03af100bb90, entries=200, sequenceid=218, filesize=38.8 K 2023-09-29T06:13:44,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/3e0e2153753440468db40372d268bdee as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/3e0e2153753440468db40372d268bdee 2023-09-29T06:13:44,044 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/3e0e2153753440468db40372d268bdee, entries=150, sequenceid=218, filesize=11.8 K 2023-09-29T06:13:44,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/85a33cebb05045c2858afc4be5c4ff81 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/85a33cebb05045c2858afc4be5c4ff81 2023-09-29T06:13:44,062 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/85a33cebb05045c2858afc4be5c4ff81, entries=150, sequenceid=218, filesize=11.8 K 2023-09-29T06:13:44,063 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 1668fccb65cfe5ce16ea12b0cc0df5e4 in 584ms, sequenceid=218, compaction requested=false 2023-09-29T06:13:44,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 1668fccb65cfe5ce16ea12b0cc0df5e4: 2023-09-29T06:13:44,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2023-09-29T06:13:44,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:44,105 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2819): Flushing 1668fccb65cfe5ce16ea12b0cc0df5e4 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2023-09-29T06:13:44,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=A 2023-09-29T06:13:44,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:44,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=B 2023-09-29T06:13:44,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:44,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=C 2023-09-29T06:13:44,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:44,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929349628894023402f87b3bcb0f1104215_1668fccb65cfe5ce16ea12b0cc0df5e4 is 50, key is test_row_1/A:col10/1695968023495/Put/seqid=0 2023-09-29T06:13:44,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:44,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. as already flushing 2023-09-29T06:13:44,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:44,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41426 deadline: 1695968084168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:44,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:44,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41454 deadline: 1695968084174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:44,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:44,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:44,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968084174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:44,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968084178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:44,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:44,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968084178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:44,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:44,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41426 deadline: 1695968084281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:44,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:44,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968084282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:44,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:44,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968084282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:44,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:44,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968084283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:44,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:44,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41454 deadline: 1695968084282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:44,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:44,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968084485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:44,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:44,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41426 deadline: 1695968084486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:44,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:44,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968084486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:44,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:44,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41454 deadline: 1695968084487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:44,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:44,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968084487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:44,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=54 2023-09-29T06:13:44,534 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929349628894023402f87b3bcb0f1104215_1668fccb65cfe5ce16ea12b0cc0df5e4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929349628894023402f87b3bcb0f1104215_1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:44,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/c6a2d88f61434d58ae80b0d4093fac9d, store: [table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:13:44,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/c6a2d88f61434d58ae80b0d4093fac9d is 175, key is test_row_1/A:col10/1695968023495/Put/seqid=0 2023-09-29T06:13:44,541 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=231, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/c6a2d88f61434d58ae80b0d4093fac9d 2023-09-29T06:13:44,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/d1c6ff4fc86d4fcfa94152d97db47d2a is 50, key is test_row_1/B:col10/1695968023495/Put/seqid=0 2023-09-29T06:13:44,566 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/d1c6ff4fc86d4fcfa94152d97db47d2a 2023-09-29T06:13:44,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/d2361f6a1d6747dbbf32477310d1540c is 50, key is test_row_1/C:col10/1695968023495/Put/seqid=0 2023-09-29T06:13:44,582 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/d2361f6a1d6747dbbf32477310d1540c 2023-09-29T06:13:44,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/c6a2d88f61434d58ae80b0d4093fac9d as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/c6a2d88f61434d58ae80b0d4093fac9d 2023-09-29T06:13:44,597 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/c6a2d88f61434d58ae80b0d4093fac9d, entries=100, sequenceid=231, filesize=21.9 K 2023-09-29T06:13:44,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/d1c6ff4fc86d4fcfa94152d97db47d2a as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/d1c6ff4fc86d4fcfa94152d97db47d2a 2023-09-29T06:13:44,604 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/d1c6ff4fc86d4fcfa94152d97db47d2a, entries=100, sequenceid=231, filesize=9.5 K 2023-09-29T06:13:44,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/d2361f6a1d6747dbbf32477310d1540c as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/d2361f6a1d6747dbbf32477310d1540c 2023-09-29T06:13:44,618 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/d2361f6a1d6747dbbf32477310d1540c, entries=100, sequenceid=231, filesize=9.5 K 2023-09-29T06:13:44,619 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3022): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=161.02 KB/164880 for 1668fccb65cfe5ce16ea12b0cc0df5e4 in 514ms, sequenceid=231, compaction requested=true 2023-09-29T06:13:44,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2520): Flush status journal for 1668fccb65cfe5ce16ea12b0cc0df5e4: 2023-09-29T06:13:44,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:44,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=55 2023-09-29T06:13:44,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=55 2023-09-29T06:13:44,621 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=55, resume processing ppid=54 2023-09-29T06:13:44,621 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=55, ppid=54, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2260 sec 2023-09-29T06:13:44,623 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=54, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 2.2330 sec 2023-09-29T06:13:44,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:44,792 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 1668fccb65cfe5ce16ea12b0cc0df5e4 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2023-09-29T06:13:44,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=A 2023-09-29T06:13:44,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:44,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=B 2023-09-29T06:13:44,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:44,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=C 2023-09-29T06:13:44,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:44,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:44,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968084793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:44,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:44,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968084794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:44,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:44,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41426 deadline: 1695968084797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:44,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:44,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968084798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:44,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:44,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41454 deadline: 1695968084798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:44,812 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309298e05a4e1388e4343b0b4090032bda0cc_1668fccb65cfe5ce16ea12b0cc0df5e4 is 50, key is test_row_0/A:col10/1695968024168/Put/seqid=0 2023-09-29T06:13:44,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:44,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968084901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:44,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:44,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968084901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:44,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:44,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968084902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:44,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:44,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41426 deadline: 1695968084902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:44,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:44,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41454 deadline: 1695968084903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:45,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:45,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:45,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968085105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:45,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41426 deadline: 1695968085106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:45,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:45,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:45,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968085106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:45,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968085106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:45,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:45,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41454 deadline: 1695968085107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:45,243 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309298e05a4e1388e4343b0b4090032bda0cc_1668fccb65cfe5ce16ea12b0cc0df5e4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309298e05a4e1388e4343b0b4090032bda0cc_1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:45,244 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/e513875edf6f4d7bb6c8f09de557474d, store: [table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:13:45,245 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/e513875edf6f4d7bb6c8f09de557474d is 175, key is test_row_0/A:col10/1695968024168/Put/seqid=0 2023-09-29T06:13:45,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:45,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968085409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:45,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:45,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41426 deadline: 1695968085409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:45,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:45,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968085411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:45,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:45,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968085415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:45,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:45,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41454 deadline: 1695968085417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:45,671 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=260, memsize=58.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/e513875edf6f4d7bb6c8f09de557474d 2023-09-29T06:13:45,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/fb4554ca49274424a35c1a468afc0c32 is 50, key is test_row_0/B:col10/1695968024168/Put/seqid=0 2023-09-29T06:13:45,715 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/fb4554ca49274424a35c1a468afc0c32 2023-09-29T06:13:45,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/e5dcea97d3c04e51b3cffa5f68ba16e4 is 50, key is test_row_0/C:col10/1695968024168/Put/seqid=0 2023-09-29T06:13:45,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:45,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41426 deadline: 1695968085914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:45,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:45,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968085915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:45,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:45,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968085917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:45,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:45,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968085920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:45,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:45,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41454 deadline: 1695968085923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:46,130 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/e5dcea97d3c04e51b3cffa5f68ba16e4 2023-09-29T06:13:46,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/e513875edf6f4d7bb6c8f09de557474d as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/e513875edf6f4d7bb6c8f09de557474d 2023-09-29T06:13:46,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/e513875edf6f4d7bb6c8f09de557474d, entries=200, sequenceid=260, filesize=38.9 K 2023-09-29T06:13:46,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/fb4554ca49274424a35c1a468afc0c32 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/fb4554ca49274424a35c1a468afc0c32 2023-09-29T06:13:46,154 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/fb4554ca49274424a35c1a468afc0c32, entries=150, sequenceid=260, filesize=11.9 K 2023-09-29T06:13:46,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/e5dcea97d3c04e51b3cffa5f68ba16e4 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/e5dcea97d3c04e51b3cffa5f68ba16e4 2023-09-29T06:13:46,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/e5dcea97d3c04e51b3cffa5f68ba16e4, entries=150, sequenceid=260, filesize=11.9 K 2023-09-29T06:13:46,166 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for 1668fccb65cfe5ce16ea12b0cc0df5e4 in 1374ms, sequenceid=260, compaction requested=true 2023-09-29T06:13:46,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 1668fccb65cfe5ce16ea12b0cc0df5e4: 2023-09-29T06:13:46,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1668fccb65cfe5ce16ea12b0cc0df5e4:A, priority=-2147483648, current under compaction store size is 1 2023-09-29T06:13:46,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:46,166 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-29T06:13:46,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1668fccb65cfe5ce16ea12b0cc0df5e4:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:13:46,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:46,167 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-29T06:13:46,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1668fccb65cfe5ce16ea12b0cc0df5e4:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:13:46,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:13:46,168 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133540 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-29T06:13:46,168 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 1668fccb65cfe5ce16ea12b0cc0df5e4/A is initiating minor compaction (all files) 2023-09-29T06:13:46,168 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1668fccb65cfe5ce16ea12b0cc0df5e4/A in TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:46,168 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/5518b8580fb84fac9a6870dc6516b3e4, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/c3e7fbfdb7744f688b3df03af100bb90, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/c6a2d88f61434d58ae80b0d4093fac9d, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/e513875edf6f4d7bb6c8f09de557474d] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp, totalSize=130.4 K 2023-09-29T06:13:46,168 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:46,168 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. files: [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/5518b8580fb84fac9a6870dc6516b3e4, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/c3e7fbfdb7744f688b3df03af100bb90, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/c6a2d88f61434d58ae80b0d4093fac9d, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/e513875edf6f4d7bb6c8f09de557474d] 2023-09-29T06:13:46,168 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46636 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-29T06:13:46,168 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 1668fccb65cfe5ce16ea12b0cc0df5e4/B is initiating minor compaction (all files) 2023-09-29T06:13:46,168 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1668fccb65cfe5ce16ea12b0cc0df5e4/B in TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:46,168 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 5518b8580fb84fac9a6870dc6516b3e4, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1695968022712 2023-09-29T06:13:46,169 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/b695634ad3fb4642ab75deaf324386c8, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/3e0e2153753440468db40372d268bdee, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/d1c6ff4fc86d4fcfa94152d97db47d2a, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/fb4554ca49274424a35c1a468afc0c32] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp, totalSize=45.5 K 2023-09-29T06:13:46,169 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting c3e7fbfdb7744f688b3df03af100bb90, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1695968022858 2023-09-29T06:13:46,169 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting b695634ad3fb4642ab75deaf324386c8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1695968022712 2023-09-29T06:13:46,169 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting c6a2d88f61434d58ae80b0d4093fac9d, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1695968023490 2023-09-29T06:13:46,170 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 3e0e2153753440468db40372d268bdee, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1695968022858 2023-09-29T06:13:46,170 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting e513875edf6f4d7bb6c8f09de557474d, keycount=200, bloomtype=ROW, size=38.9 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1695968024162 2023-09-29T06:13:46,170 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting d1c6ff4fc86d4fcfa94152d97db47d2a, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1695968023490 2023-09-29T06:13:46,171 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting fb4554ca49274424a35c1a468afc0c32, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1695968024162 2023-09-29T06:13:46,188 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1668fccb65cfe5ce16ea12b0cc0df5e4#B#compaction#181 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:46,188 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:13:46,188 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/841bf539ffad45d28bad0935f07d5ef1 is 50, key is test_row_0/B:col10/1695968024168/Put/seqid=0 2023-09-29T06:13:46,194 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20230929121fe7621d384dbd953e939a21cb8b22_1668fccb65cfe5ce16ea12b0cc0df5e4 store=[table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:13:46,195 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20230929121fe7621d384dbd953e939a21cb8b22_1668fccb65cfe5ce16ea12b0cc0df5e4, store=[table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:13:46,195 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929121fe7621d384dbd953e939a21cb8b22_1668fccb65cfe5ce16ea12b0cc0df5e4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:13:46,222 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1668fccb65cfe5ce16ea12b0cc0df5e4#A#compaction#182 average throughput is 0.72 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:46,223 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/f1a4b3fe2d454dfbbf312ed533c92654 is 175, key is test_row_0/A:col10/1695968024168/Put/seqid=0 2023-09-29T06:13:46,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=54 2023-09-29T06:13:46,499 INFO [Thread-842 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 54 completed 2023-09-29T06:13:46,501 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:13:46,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2023-09-29T06:13:46,504 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:13:46,505 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:13:46,505 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:13:46,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=56 2023-09-29T06:13:46,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=56 2023-09-29T06:13:46,630 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/841bf539ffad45d28bad0935f07d5ef1 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/841bf539ffad45d28bad0935f07d5ef1 2023-09-29T06:13:46,636 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 1668fccb65cfe5ce16ea12b0cc0df5e4/B of 1668fccb65cfe5ce16ea12b0cc0df5e4 into 841bf539ffad45d28bad0935f07d5ef1(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:46,636 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1668fccb65cfe5ce16ea12b0cc0df5e4: 2023-09-29T06:13:46,636 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4., storeName=1668fccb65cfe5ce16ea12b0cc0df5e4/B, priority=12, startTime=1695968026166; duration=0sec 2023-09-29T06:13:46,636 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:13:46,636 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1668fccb65cfe5ce16ea12b0cc0df5e4:B 2023-09-29T06:13:46,636 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-29T06:13:46,637 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/f1a4b3fe2d454dfbbf312ed533c92654 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/f1a4b3fe2d454dfbbf312ed533c92654 2023-09-29T06:13:46,639 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46636 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-29T06:13:46,639 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 1668fccb65cfe5ce16ea12b0cc0df5e4/C is initiating minor compaction (all files) 2023-09-29T06:13:46,639 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1668fccb65cfe5ce16ea12b0cc0df5e4/C in TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:46,639 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/2c0cf557040b4fa5911c8bc1f640ec85, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/85a33cebb05045c2858afc4be5c4ff81, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/d2361f6a1d6747dbbf32477310d1540c, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/e5dcea97d3c04e51b3cffa5f68ba16e4] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp, totalSize=45.5 K 2023-09-29T06:13:46,640 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 2c0cf557040b4fa5911c8bc1f640ec85, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1695968022712 2023-09-29T06:13:46,640 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 85a33cebb05045c2858afc4be5c4ff81, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1695968022858 2023-09-29T06:13:46,641 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting d2361f6a1d6747dbbf32477310d1540c, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1695968023490 2023-09-29T06:13:46,642 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting e5dcea97d3c04e51b3cffa5f68ba16e4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1695968024162 2023-09-29T06:13:46,645 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 1668fccb65cfe5ce16ea12b0cc0df5e4/A of 1668fccb65cfe5ce16ea12b0cc0df5e4 into f1a4b3fe2d454dfbbf312ed533c92654(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:46,645 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1668fccb65cfe5ce16ea12b0cc0df5e4: 2023-09-29T06:13:46,645 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4., storeName=1668fccb65cfe5ce16ea12b0cc0df5e4/A, priority=12, startTime=1695968026166; duration=0sec 2023-09-29T06:13:46,646 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:46,646 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1668fccb65cfe5ce16ea12b0cc0df5e4:A 2023-09-29T06:13:46,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2023-09-29T06:13:46,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:46,658 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2819): Flushing 1668fccb65cfe5ce16ea12b0cc0df5e4 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2023-09-29T06:13:46,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=A 2023-09-29T06:13:46,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:46,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=B 2023-09-29T06:13:46,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:46,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=C 2023-09-29T06:13:46,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:46,667 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1668fccb65cfe5ce16ea12b0cc0df5e4#C#compaction#183 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:46,668 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/45fee92478294bdaa44fcbb37c41a211 is 50, key is test_row_0/C:col10/1695968024168/Put/seqid=0 2023-09-29T06:13:46,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929024c14c243664a30b22c86ffcbc8eb1e_1668fccb65cfe5ce16ea12b0cc0df5e4 is 50, key is test_row_1/A:col10/1695968024797/Put/seqid=0 2023-09-29T06:13:46,717 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/45fee92478294bdaa44fcbb37c41a211 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/45fee92478294bdaa44fcbb37c41a211 2023-09-29T06:13:46,729 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 1668fccb65cfe5ce16ea12b0cc0df5e4/C of 1668fccb65cfe5ce16ea12b0cc0df5e4 into 45fee92478294bdaa44fcbb37c41a211(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:46,729 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1668fccb65cfe5ce16ea12b0cc0df5e4: 2023-09-29T06:13:46,729 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4., storeName=1668fccb65cfe5ce16ea12b0cc0df5e4/C, priority=12, startTime=1695968026167; duration=0sec 2023-09-29T06:13:46,729 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:46,729 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1668fccb65cfe5ce16ea12b0cc0df5e4:C 2023-09-29T06:13:46,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=56 2023-09-29T06:13:46,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:46,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. as already flushing 2023-09-29T06:13:47,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:47,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:47,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968087007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:47,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41454 deadline: 1695968087007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:47,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:47,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968087008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:47,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:47,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968087011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:47,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:47,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41426 deadline: 1695968087011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:47,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:47,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968087112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:47,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:47,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41454 deadline: 1695968087112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:47,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:47,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968087112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:47,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:47,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968087114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:47,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:47,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41426 deadline: 1695968087114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:47,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=56 2023-09-29T06:13:47,126 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929024c14c243664a30b22c86ffcbc8eb1e_1668fccb65cfe5ce16ea12b0cc0df5e4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929024c14c243664a30b22c86ffcbc8eb1e_1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:47,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/5d1decb820d9406dbf9e60cdb2c05399, store: [table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:13:47,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/5d1decb820d9406dbf9e60cdb2c05399 is 175, key is test_row_1/A:col10/1695968024797/Put/seqid=0 2023-09-29T06:13:47,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:47,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968087315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:47,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:47,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41454 deadline: 1695968087315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:47,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:47,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968087315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:47,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:47,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41426 deadline: 1695968087316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:47,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:47,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968087317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:47,560 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=269, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/5d1decb820d9406dbf9e60cdb2c05399 2023-09-29T06:13:47,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/8e897e1875974cd59bab8c4abf91f6e5 is 50, key is test_row_1/B:col10/1695968024797/Put/seqid=0 2023-09-29T06:13:47,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=56 2023-09-29T06:13:47,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:47,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968087618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:47,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:47,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41454 deadline: 1695968087619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:47,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:47,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968087620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:47,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:47,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968087620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:47,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:47,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41426 deadline: 1695968087621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:47,987 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/8e897e1875974cd59bab8c4abf91f6e5 2023-09-29T06:13:47,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/546d55e150e349ffa40a977db87d14ff is 50, key is test_row_1/C:col10/1695968024797/Put/seqid=0 2023-09-29T06:13:48,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:48,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968088122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:48,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:48,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968088122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:48,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:48,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968088123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:48,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:48,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41454 deadline: 1695968088126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:48,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:48,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41426 deadline: 1695968088130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:48,410 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/546d55e150e349ffa40a977db87d14ff 2023-09-29T06:13:48,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/5d1decb820d9406dbf9e60cdb2c05399 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/5d1decb820d9406dbf9e60cdb2c05399 2023-09-29T06:13:48,422 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/5d1decb820d9406dbf9e60cdb2c05399, entries=50, sequenceid=269, filesize=13.5 K 2023-09-29T06:13:48,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/8e897e1875974cd59bab8c4abf91f6e5 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/8e897e1875974cd59bab8c4abf91f6e5 2023-09-29T06:13:48,440 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/8e897e1875974cd59bab8c4abf91f6e5, entries=50, sequenceid=269, filesize=7.2 K 2023-09-29T06:13:48,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/546d55e150e349ffa40a977db87d14ff as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/546d55e150e349ffa40a977db87d14ff 2023-09-29T06:13:48,454 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/546d55e150e349ffa40a977db87d14ff, entries=50, sequenceid=269, filesize=7.2 K 2023-09-29T06:13:48,455 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3022): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=181.14 KB/185490 for 1668fccb65cfe5ce16ea12b0cc0df5e4 in 1797ms, sequenceid=269, compaction requested=false 2023-09-29T06:13:48,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2520): Flush status journal for 1668fccb65cfe5ce16ea12b0cc0df5e4: 2023-09-29T06:13:48,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:48,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=57 2023-09-29T06:13:48,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=57 2023-09-29T06:13:48,458 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=57, resume processing ppid=56 2023-09-29T06:13:48,458 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=57, ppid=56, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9510 sec 2023-09-29T06:13:48,459 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=56, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 1.9570 sec 2023-09-29T06:13:48,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=56 2023-09-29T06:13:48,619 INFO [Thread-842 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 56 completed 2023-09-29T06:13:48,621 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:13:48,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2023-09-29T06:13:48,624 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:13:48,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=58 2023-09-29T06:13:48,625 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:13:48,625 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:13:48,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=58 2023-09-29T06:13:48,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2023-09-29T06:13:48,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:48,779 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2819): Flushing 1668fccb65cfe5ce16ea12b0cc0df5e4 3/3 column families, dataSize=181.14 KB heapSize=475.36 KB 2023-09-29T06:13:48,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=A 2023-09-29T06:13:48,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:48,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=B 2023-09-29T06:13:48,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:48,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=C 2023-09-29T06:13:48,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:48,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309294c3e8997257b4371a82a512877215d14_1668fccb65cfe5ce16ea12b0cc0df5e4 is 50, key is test_row_0/A:col10/1695968027009/Put/seqid=0 2023-09-29T06:13:48,822 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309294c3e8997257b4371a82a512877215d14_1668fccb65cfe5ce16ea12b0cc0df5e4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309294c3e8997257b4371a82a512877215d14_1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:48,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/39a5ed35e50245d494641c032d3bd216, store: [table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:13:48,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/39a5ed35e50245d494641c032d3bd216 is 175, key is test_row_0/A:col10/1695968027009/Put/seqid=0 2023-09-29T06:13:48,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=58 2023-09-29T06:13:49,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. as already flushing 2023-09-29T06:13:49,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:49,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:49,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41454 deadline: 1695968089135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:49,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:49,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968089136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:49,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:49,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41426 deadline: 1695968089139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:49,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:49,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968089142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:49,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:49,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968089142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:49,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=58 2023-09-29T06:13:49,230 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=300, memsize=60.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/39a5ed35e50245d494641c032d3bd216 2023-09-29T06:13:49,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/2ca492b23399499abf9424f420d19b75 is 50, key is test_row_0/B:col10/1695968027009/Put/seqid=0 2023-09-29T06:13:49,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:49,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968089242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:49,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:49,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968089248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:49,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:49,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968089248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:49,260 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/2ca492b23399499abf9424f420d19b75 2023-09-29T06:13:49,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/c84d8d7af6564a58bf99ad0be1940060 is 50, key is test_row_0/C:col10/1695968027009/Put/seqid=0 2023-09-29T06:13:49,312 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/c84d8d7af6564a58bf99ad0be1940060 2023-09-29T06:13:49,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/39a5ed35e50245d494641c032d3bd216 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/39a5ed35e50245d494641c032d3bd216 2023-09-29T06:13:49,325 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/39a5ed35e50245d494641c032d3bd216, entries=150, sequenceid=300, filesize=30.5 K 2023-09-29T06:13:49,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/2ca492b23399499abf9424f420d19b75 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/2ca492b23399499abf9424f420d19b75 2023-09-29T06:13:49,332 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/2ca492b23399499abf9424f420d19b75, entries=150, sequenceid=300, filesize=12.0 K 2023-09-29T06:13:49,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/c84d8d7af6564a58bf99ad0be1940060 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/c84d8d7af6564a58bf99ad0be1940060 2023-09-29T06:13:49,346 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/c84d8d7af6564a58bf99ad0be1940060, entries=150, sequenceid=300, filesize=12.0 K 2023-09-29T06:13:49,347 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3022): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=26.84 KB/27480 for 1668fccb65cfe5ce16ea12b0cc0df5e4 in 568ms, sequenceid=300, compaction requested=true 2023-09-29T06:13:49,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2520): Flush status journal for 1668fccb65cfe5ce16ea12b0cc0df5e4: 2023-09-29T06:13:49,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:49,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=59 2023-09-29T06:13:49,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=59 2023-09-29T06:13:49,352 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=59, resume processing ppid=58 2023-09-29T06:13:49,352 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=59, ppid=58, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 724 msec 2023-09-29T06:13:49,355 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=58, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 732 msec 2023-09-29T06:13:49,457 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 1668fccb65cfe5ce16ea12b0cc0df5e4 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2023-09-29T06:13:49,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:49,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=A 2023-09-29T06:13:49,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:49,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=B 2023-09-29T06:13:49,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:49,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=C 2023-09-29T06:13:49,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:49,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:49,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968089486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:49,491 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309299b3d156444d44c9a91d8ecd9d6a5128d_1668fccb65cfe5ce16ea12b0cc0df5e4 is 50, key is test_row_0/A:col10/1695968029454/Put/seqid=0 2023-09-29T06:13:49,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:49,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968089489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:49,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:49,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968089489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:49,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:49,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968089596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:49,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:49,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968089596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:49,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:49,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968089596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:49,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=58 2023-09-29T06:13:49,729 INFO [Thread-842 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 58 completed 2023-09-29T06:13:49,731 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:13:49,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2023-09-29T06:13:49,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=60 2023-09-29T06:13:49,734 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:13:49,735 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:13:49,735 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:13:49,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:49,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968089799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:49,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:49,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968089799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:49,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:49,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968089800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:49,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=60 2023-09-29T06:13:49,890 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2023-09-29T06:13:49,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:49,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. as already flushing 2023-09-29T06:13:49,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:49,891 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(53): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:49,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:49,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:49,906 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309299b3d156444d44c9a91d8ecd9d6a5128d_1668fccb65cfe5ce16ea12b0cc0df5e4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309299b3d156444d44c9a91d8ecd9d6a5128d_1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:49,907 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/2d64ebd9fd554a2782b9d17e6eb836d8, store: [table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:13:49,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/2d64ebd9fd554a2782b9d17e6eb836d8 is 175, key is test_row_0/A:col10/1695968029454/Put/seqid=0 2023-09-29T06:13:49,922 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=313, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/2d64ebd9fd554a2782b9d17e6eb836d8 2023-09-29T06:13:49,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/34bd0b053b8a4f468f1fa1773f49b285 is 50, key is test_row_0/B:col10/1695968029454/Put/seqid=0 2023-09-29T06:13:50,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=60 2023-09-29T06:13:50,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2023-09-29T06:13:50,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:50,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. as already flushing 2023-09-29T06:13:50,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:50,044 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(53): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:50,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:50,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:50,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:50,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968090101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:50,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:50,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968090101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:50,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:50,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968090101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:50,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2023-09-29T06:13:50,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:50,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. as already flushing 2023-09-29T06:13:50,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:50,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(53): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:50,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:50,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:50,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=60 2023-09-29T06:13:50,344 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/34bd0b053b8a4f468f1fa1773f49b285 2023-09-29T06:13:50,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2023-09-29T06:13:50,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:50,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. as already flushing 2023-09-29T06:13:50,351 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/12b698a8f39a4ae99c9a11379f5c8eee is 50, key is test_row_0/C:col10/1695968029454/Put/seqid=0 2023-09-29T06:13:50,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:50,351 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(53): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:50,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:50,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:50,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/12b698a8f39a4ae99c9a11379f5c8eee 2023-09-29T06:13:50,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/2d64ebd9fd554a2782b9d17e6eb836d8 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/2d64ebd9fd554a2782b9d17e6eb836d8 2023-09-29T06:13:50,374 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/2d64ebd9fd554a2782b9d17e6eb836d8, entries=150, sequenceid=313, filesize=30.5 K 2023-09-29T06:13:50,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/34bd0b053b8a4f468f1fa1773f49b285 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/34bd0b053b8a4f468f1fa1773f49b285 2023-09-29T06:13:50,383 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/34bd0b053b8a4f468f1fa1773f49b285, entries=150, sequenceid=313, filesize=12.0 K 2023-09-29T06:13:50,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/12b698a8f39a4ae99c9a11379f5c8eee as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/12b698a8f39a4ae99c9a11379f5c8eee 2023-09-29T06:13:50,389 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/12b698a8f39a4ae99c9a11379f5c8eee, entries=150, sequenceid=313, filesize=12.0 K 2023-09-29T06:13:50,390 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 1668fccb65cfe5ce16ea12b0cc0df5e4 in 933ms, sequenceid=313, compaction requested=true 2023-09-29T06:13:50,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 1668fccb65cfe5ce16ea12b0cc0df5e4: 2023-09-29T06:13:50,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1668fccb65cfe5ce16ea12b0cc0df5e4:A, priority=-2147483648, current under compaction store size is 1 2023-09-29T06:13:50,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:50,390 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-29T06:13:50,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1668fccb65cfe5ce16ea12b0cc0df5e4:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:13:50,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:50,390 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-29T06:13:50,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1668fccb65cfe5ce16ea12b0cc0df5e4:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:13:50,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:13:50,393 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 108042 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-29T06:13:50,393 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 1668fccb65cfe5ce16ea12b0cc0df5e4/A is initiating minor compaction (all files) 2023-09-29T06:13:50,393 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1668fccb65cfe5ce16ea12b0cc0df5e4/A in TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:50,393 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 44730 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-29T06:13:50,393 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/f1a4b3fe2d454dfbbf312ed533c92654, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/5d1decb820d9406dbf9e60cdb2c05399, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/39a5ed35e50245d494641c032d3bd216, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/2d64ebd9fd554a2782b9d17e6eb836d8] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp, totalSize=105.5 K 2023-09-29T06:13:50,393 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 1668fccb65cfe5ce16ea12b0cc0df5e4/B is initiating minor compaction (all files) 2023-09-29T06:13:50,393 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:50,393 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1668fccb65cfe5ce16ea12b0cc0df5e4/B in TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:50,393 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. files: [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/f1a4b3fe2d454dfbbf312ed533c92654, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/5d1decb820d9406dbf9e60cdb2c05399, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/39a5ed35e50245d494641c032d3bd216, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/2d64ebd9fd554a2782b9d17e6eb836d8] 2023-09-29T06:13:50,393 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/841bf539ffad45d28bad0935f07d5ef1, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/8e897e1875974cd59bab8c4abf91f6e5, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/2ca492b23399499abf9424f420d19b75, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/34bd0b053b8a4f468f1fa1773f49b285] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp, totalSize=43.7 K 2023-09-29T06:13:50,394 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting f1a4b3fe2d454dfbbf312ed533c92654, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1695968024162 2023-09-29T06:13:50,394 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 841bf539ffad45d28bad0935f07d5ef1, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1695968024162 2023-09-29T06:13:50,394 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 5d1decb820d9406dbf9e60cdb2c05399, keycount=50, bloomtype=ROW, size=13.5 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1695968024797 2023-09-29T06:13:50,395 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 8e897e1875974cd59bab8c4abf91f6e5, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1695968024797 2023-09-29T06:13:50,395 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 39a5ed35e50245d494641c032d3bd216, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1695968027005 2023-09-29T06:13:50,395 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 2ca492b23399499abf9424f420d19b75, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1695968027005 2023-09-29T06:13:50,395 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 2d64ebd9fd554a2782b9d17e6eb836d8, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1695968029448 2023-09-29T06:13:50,396 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 34bd0b053b8a4f468f1fa1773f49b285, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1695968029448 2023-09-29T06:13:50,408 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:13:50,408 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1668fccb65cfe5ce16ea12b0cc0df5e4#B#compaction#193 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:50,409 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/dce408cf71f74e999c731d8d8d54847c is 50, key is test_row_0/B:col10/1695968029454/Put/seqid=0 2023-09-29T06:13:50,410 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20230929fc2926a080bf4c22887a405d63521c4a_1668fccb65cfe5ce16ea12b0cc0df5e4 store=[table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:13:50,412 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20230929fc2926a080bf4c22887a405d63521c4a_1668fccb65cfe5ce16ea12b0cc0df5e4, store=[table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:13:50,412 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929fc2926a080bf4c22887a405d63521c4a_1668fccb65cfe5ce16ea12b0cc0df5e4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:13:50,440 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1668fccb65cfe5ce16ea12b0cc0df5e4#A#compaction#194 average throughput is 0.76 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:50,440 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/be6a80eb693a4722840efe2720a394e9 is 175, key is test_row_0/A:col10/1695968029454/Put/seqid=0 2023-09-29T06:13:50,450 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/be6a80eb693a4722840efe2720a394e9 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/be6a80eb693a4722840efe2720a394e9 2023-09-29T06:13:50,455 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 1668fccb65cfe5ce16ea12b0cc0df5e4/A of 1668fccb65cfe5ce16ea12b0cc0df5e4 into be6a80eb693a4722840efe2720a394e9(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:50,455 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1668fccb65cfe5ce16ea12b0cc0df5e4: 2023-09-29T06:13:50,455 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4., storeName=1668fccb65cfe5ce16ea12b0cc0df5e4/A, priority=12, startTime=1695968030390; duration=0sec 2023-09-29T06:13:50,455 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:13:50,455 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1668fccb65cfe5ce16ea12b0cc0df5e4:A 2023-09-29T06:13:50,455 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-29T06:13:50,456 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 44730 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-29T06:13:50,457 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 1668fccb65cfe5ce16ea12b0cc0df5e4/C is initiating minor compaction (all files) 2023-09-29T06:13:50,457 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1668fccb65cfe5ce16ea12b0cc0df5e4/C in TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:50,457 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/45fee92478294bdaa44fcbb37c41a211, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/546d55e150e349ffa40a977db87d14ff, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/c84d8d7af6564a58bf99ad0be1940060, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/12b698a8f39a4ae99c9a11379f5c8eee] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp, totalSize=43.7 K 2023-09-29T06:13:50,457 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 45fee92478294bdaa44fcbb37c41a211, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1695968024162 2023-09-29T06:13:50,462 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 546d55e150e349ffa40a977db87d14ff, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1695968024797 2023-09-29T06:13:50,463 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting c84d8d7af6564a58bf99ad0be1940060, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1695968027005 2023-09-29T06:13:50,463 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 12b698a8f39a4ae99c9a11379f5c8eee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1695968029448 2023-09-29T06:13:50,472 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1668fccb65cfe5ce16ea12b0cc0df5e4#C#compaction#195 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:50,473 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/daf0bebb32d5458eb99f97b12f0cf986 is 50, key is test_row_0/C:col10/1695968029454/Put/seqid=0 2023-09-29T06:13:50,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2023-09-29T06:13:50,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:50,506 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2819): Flushing 1668fccb65cfe5ce16ea12b0cc0df5e4 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2023-09-29T06:13:50,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=A 2023-09-29T06:13:50,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:50,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=B 2023-09-29T06:13:50,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:50,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=C 2023-09-29T06:13:50,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:50,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023092920016fa647d34fcf8ade822f8c7de576_1668fccb65cfe5ce16ea12b0cc0df5e4 is 50, key is test_row_0/A:col10/1695968029486/Put/seqid=0 2023-09-29T06:13:50,535 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023092920016fa647d34fcf8ade822f8c7de576_1668fccb65cfe5ce16ea12b0cc0df5e4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023092920016fa647d34fcf8ade822f8c7de576_1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:50,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/7e274237aa6d4577be8b839cfdd64604, store: [table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:13:50,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/7e274237aa6d4577be8b839cfdd64604 is 175, key is test_row_0/A:col10/1695968029486/Put/seqid=0 2023-09-29T06:13:50,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. as already flushing 2023-09-29T06:13:50,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:50,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:50,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968090622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:50,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:50,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968090622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:50,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:50,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968090625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:50,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:50,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968090726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:50,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:50,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968090727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:50,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:50,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968090729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:50,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=60 2023-09-29T06:13:50,843 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/dce408cf71f74e999c731d8d8d54847c as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/dce408cf71f74e999c731d8d8d54847c 2023-09-29T06:13:50,849 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 1668fccb65cfe5ce16ea12b0cc0df5e4/B of 1668fccb65cfe5ce16ea12b0cc0df5e4 into dce408cf71f74e999c731d8d8d54847c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:50,849 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1668fccb65cfe5ce16ea12b0cc0df5e4: 2023-09-29T06:13:50,849 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4., storeName=1668fccb65cfe5ce16ea12b0cc0df5e4/B, priority=12, startTime=1695968030390; duration=0sec 2023-09-29T06:13:50,849 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:50,849 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1668fccb65cfe5ce16ea12b0cc0df5e4:B 2023-09-29T06:13:50,883 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/daf0bebb32d5458eb99f97b12f0cf986 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/daf0bebb32d5458eb99f97b12f0cf986 2023-09-29T06:13:50,888 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 1668fccb65cfe5ce16ea12b0cc0df5e4/C of 1668fccb65cfe5ce16ea12b0cc0df5e4 into daf0bebb32d5458eb99f97b12f0cf986(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:50,889 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1668fccb65cfe5ce16ea12b0cc0df5e4: 2023-09-29T06:13:50,889 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4., storeName=1668fccb65cfe5ce16ea12b0cc0df5e4/C, priority=12, startTime=1695968030390; duration=0sec 2023-09-29T06:13:50,889 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:50,889 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1668fccb65cfe5ce16ea12b0cc0df5e4:C 2023-09-29T06:13:50,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:50,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968090929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:50,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:50,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968090930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:50,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:50,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968090933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:50,951 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=337, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/7e274237aa6d4577be8b839cfdd64604 2023-09-29T06:13:50,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/2da76a3ea0c44799bfa6bc70d6e509d0 is 50, key is test_row_0/B:col10/1695968029486/Put/seqid=0 2023-09-29T06:13:50,971 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/2da76a3ea0c44799bfa6bc70d6e509d0 2023-09-29T06:13:50,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/064e9fb169f24dcaa50e021fc1207a94 is 50, key is test_row_0/C:col10/1695968029486/Put/seqid=0 2023-09-29T06:13:51,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:51,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41454 deadline: 1695968091152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:51,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:51,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41426 deadline: 1695968091156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:51,158 DEBUG [Thread-832 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4146 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_0' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4., hostname=jenkins-hbase3.apache.org,39783,1695967978138, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:572) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:559) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:197) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:218) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:750) 2023-09-29T06:13:51,164 DEBUG [Thread-834 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4155 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_2' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4., hostname=jenkins-hbase3.apache.org,39783,1695967978138, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:572) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:559) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:197) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:218) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:750) 2023-09-29T06:13:51,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:51,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968091231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:51,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:51,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968091233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:51,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:51,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968091236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:51,395 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/064e9fb169f24dcaa50e021fc1207a94 2023-09-29T06:13:51,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/7e274237aa6d4577be8b839cfdd64604 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/7e274237aa6d4577be8b839cfdd64604 2023-09-29T06:13:51,413 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/7e274237aa6d4577be8b839cfdd64604, entries=150, sequenceid=337, filesize=30.5 K 2023-09-29T06:13:51,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/2da76a3ea0c44799bfa6bc70d6e509d0 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/2da76a3ea0c44799bfa6bc70d6e509d0 2023-09-29T06:13:51,418 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/2da76a3ea0c44799bfa6bc70d6e509d0, entries=150, sequenceid=337, filesize=12.0 K 2023-09-29T06:13:51,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/064e9fb169f24dcaa50e021fc1207a94 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/064e9fb169f24dcaa50e021fc1207a94 2023-09-29T06:13:51,433 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/064e9fb169f24dcaa50e021fc1207a94, entries=150, sequenceid=337, filesize=12.0 K 2023-09-29T06:13:51,434 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3022): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 1668fccb65cfe5ce16ea12b0cc0df5e4 in 928ms, sequenceid=337, compaction requested=false 2023-09-29T06:13:51,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2520): Flush status journal for 1668fccb65cfe5ce16ea12b0cc0df5e4: 2023-09-29T06:13:51,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:51,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=61 2023-09-29T06:13:51,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=61 2023-09-29T06:13:51,437 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=61, resume processing ppid=60 2023-09-29T06:13:51,437 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=61, ppid=60, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7010 sec 2023-09-29T06:13:51,439 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=60, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 1.7070 sec 2023-09-29T06:13:51,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:51,737 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 1668fccb65cfe5ce16ea12b0cc0df5e4 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2023-09-29T06:13:51,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=A 2023-09-29T06:13:51,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:51,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=B 2023-09-29T06:13:51,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:51,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=C 2023-09-29T06:13:51,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:51,752 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929b7c44e0e0fce41cd89decd5d5cde05b7_1668fccb65cfe5ce16ea12b0cc0df5e4 is 50, key is test_row_0/A:col10/1695968030616/Put/seqid=0 2023-09-29T06:13:51,778 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929b7c44e0e0fce41cd89decd5d5cde05b7_1668fccb65cfe5ce16ea12b0cc0df5e4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929b7c44e0e0fce41cd89decd5d5cde05b7_1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:51,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:51,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968091774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:51,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:51,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:51,782 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/213cc87396074545a7ce164788e9e9b7, store: [table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:13:51,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968091777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:51,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968091779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:51,783 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/213cc87396074545a7ce164788e9e9b7 is 175, key is test_row_0/A:col10/1695968030616/Put/seqid=0 2023-09-29T06:13:51,812 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=354, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/213cc87396074545a7ce164788e9e9b7 2023-09-29T06:13:51,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/ea9180622cf042cda8999f5de5e0cf97 is 50, key is test_row_0/B:col10/1695968030616/Put/seqid=0 2023-09-29T06:13:51,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=60 2023-09-29T06:13:51,839 INFO [Thread-842 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 60 completed 2023-09-29T06:13:51,841 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:13:51,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2023-09-29T06:13:51,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=62 2023-09-29T06:13:51,858 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:13:51,859 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:13:51,863 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:13:51,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:51,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968091884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:51,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:51,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968091884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:51,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:51,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968091891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:51,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=62 2023-09-29T06:13:52,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2023-09-29T06:13:52,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:52,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. as already flushing 2023-09-29T06:13:52,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:52,017 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(53): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:52,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:52,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:52,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:52,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968092086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:52,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:52,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968092087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:52,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:52,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968092096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:52,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=62 2023-09-29T06:13:52,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2023-09-29T06:13:52,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:52,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. as already flushing 2023-09-29T06:13:52,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:52,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(53): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:52,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:52,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:52,271 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/ea9180622cf042cda8999f5de5e0cf97 2023-09-29T06:13:52,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/cbabfb10069f48d0a50c5452ea1689e6 is 50, key is test_row_0/C:col10/1695968030616/Put/seqid=0 2023-09-29T06:13:52,298 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/cbabfb10069f48d0a50c5452ea1689e6 2023-09-29T06:13:52,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/213cc87396074545a7ce164788e9e9b7 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/213cc87396074545a7ce164788e9e9b7 2023-09-29T06:13:52,311 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/213cc87396074545a7ce164788e9e9b7, entries=200, sequenceid=354, filesize=39.0 K 2023-09-29T06:13:52,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/ea9180622cf042cda8999f5de5e0cf97 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/ea9180622cf042cda8999f5de5e0cf97 2023-09-29T06:13:52,319 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/ea9180622cf042cda8999f5de5e0cf97, entries=150, sequenceid=354, filesize=12.0 K 2023-09-29T06:13:52,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/cbabfb10069f48d0a50c5452ea1689e6 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/cbabfb10069f48d0a50c5452ea1689e6 2023-09-29T06:13:52,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2023-09-29T06:13:52,326 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/cbabfb10069f48d0a50c5452ea1689e6, entries=150, sequenceid=354, filesize=12.0 K 2023-09-29T06:13:52,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:52,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. as already flushing 2023-09-29T06:13:52,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:52,326 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(53): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:52,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:52,327 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 1668fccb65cfe5ce16ea12b0cc0df5e4 in 590ms, sequenceid=354, compaction requested=true 2023-09-29T06:13:52,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 1668fccb65cfe5ce16ea12b0cc0df5e4: 2023-09-29T06:13:52,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1668fccb65cfe5ce16ea12b0cc0df5e4:A, priority=-2147483648, current under compaction store size is 1 2023-09-29T06:13:52,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:52,327 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:13:52,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:52,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1668fccb65cfe5ce16ea12b0cc0df5e4:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:13:52,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:13:52,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1668fccb65cfe5ce16ea12b0cc0df5e4:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:13:52,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:13:52,327 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:13:52,329 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103124 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:13:52,329 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 1668fccb65cfe5ce16ea12b0cc0df5e4/A is initiating minor compaction (all files) 2023-09-29T06:13:52,329 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1668fccb65cfe5ce16ea12b0cc0df5e4/A in TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:52,330 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/be6a80eb693a4722840efe2720a394e9, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/7e274237aa6d4577be8b839cfdd64604, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/213cc87396074545a7ce164788e9e9b7] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp, totalSize=100.7 K 2023-09-29T06:13:52,330 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:52,330 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. files: [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/be6a80eb693a4722840efe2720a394e9, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/7e274237aa6d4577be8b839cfdd64604, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/213cc87396074545a7ce164788e9e9b7] 2023-09-29T06:13:52,331 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting be6a80eb693a4722840efe2720a394e9, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1695968029448 2023-09-29T06:13:52,331 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 7e274237aa6d4577be8b839cfdd64604, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1695968029481 2023-09-29T06:13:52,331 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37568 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:13:52,331 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 1668fccb65cfe5ce16ea12b0cc0df5e4/B is initiating minor compaction (all files) 2023-09-29T06:13:52,331 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1668fccb65cfe5ce16ea12b0cc0df5e4/B in TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:52,332 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/dce408cf71f74e999c731d8d8d54847c, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/2da76a3ea0c44799bfa6bc70d6e509d0, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/ea9180622cf042cda8999f5de5e0cf97] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp, totalSize=36.7 K 2023-09-29T06:13:52,332 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 213cc87396074545a7ce164788e9e9b7, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1695968030616 2023-09-29T06:13:52,333 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting dce408cf71f74e999c731d8d8d54847c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1695968029448 2023-09-29T06:13:52,334 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 2da76a3ea0c44799bfa6bc70d6e509d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1695968029481 2023-09-29T06:13:52,342 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting ea9180622cf042cda8999f5de5e0cf97, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1695968030616 2023-09-29T06:13:52,354 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:13:52,358 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202309295add3372019f4bdca0a058ec26e386c1_1668fccb65cfe5ce16ea12b0cc0df5e4 store=[table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:13:52,359 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1668fccb65cfe5ce16ea12b0cc0df5e4#B#compaction#203 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:52,360 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/0a7aca788ab34c5797fdc9a787b0edb7 is 50, key is test_row_0/B:col10/1695968030616/Put/seqid=0 2023-09-29T06:13:52,360 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202309295add3372019f4bdca0a058ec26e386c1_1668fccb65cfe5ce16ea12b0cc0df5e4, store=[table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:13:52,360 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309295add3372019f4bdca0a058ec26e386c1_1668fccb65cfe5ce16ea12b0cc0df5e4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:13:52,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:52,394 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 1668fccb65cfe5ce16ea12b0cc0df5e4 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2023-09-29T06:13:52,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=A 2023-09-29T06:13:52,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:52,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=B 2023-09-29T06:13:52,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:52,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=C 2023-09-29T06:13:52,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:52,397 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1668fccb65cfe5ce16ea12b0cc0df5e4#A#compaction#202 average throughput is 0.57 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:52,397 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/a0bfabd2be26417e9ca5684a07c473bf is 175, key is test_row_0/A:col10/1695968030616/Put/seqid=0 2023-09-29T06:13:52,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:52,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968092412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:52,415 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/0a7aca788ab34c5797fdc9a787b0edb7 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/0a7aca788ab34c5797fdc9a787b0edb7 2023-09-29T06:13:52,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:52,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968092414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:52,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:52,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968092415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:52,426 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309299049f4950f624bc49cf365964506ef49_1668fccb65cfe5ce16ea12b0cc0df5e4 is 50, key is test_row_0/A:col10/1695968031775/Put/seqid=0 2023-09-29T06:13:52,434 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 1668fccb65cfe5ce16ea12b0cc0df5e4/B of 1668fccb65cfe5ce16ea12b0cc0df5e4 into 0a7aca788ab34c5797fdc9a787b0edb7(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:52,434 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1668fccb65cfe5ce16ea12b0cc0df5e4: 2023-09-29T06:13:52,434 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4., storeName=1668fccb65cfe5ce16ea12b0cc0df5e4/B, priority=13, startTime=1695968032327; duration=0sec 2023-09-29T06:13:52,434 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:13:52,434 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1668fccb65cfe5ce16ea12b0cc0df5e4:B 2023-09-29T06:13:52,434 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:13:52,435 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37568 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:13:52,435 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 1668fccb65cfe5ce16ea12b0cc0df5e4/C is initiating minor compaction (all files) 2023-09-29T06:13:52,436 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1668fccb65cfe5ce16ea12b0cc0df5e4/C in TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:52,436 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/daf0bebb32d5458eb99f97b12f0cf986, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/064e9fb169f24dcaa50e021fc1207a94, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/cbabfb10069f48d0a50c5452ea1689e6] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp, totalSize=36.7 K 2023-09-29T06:13:52,436 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting daf0bebb32d5458eb99f97b12f0cf986, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1695968029448 2023-09-29T06:13:52,436 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 064e9fb169f24dcaa50e021fc1207a94, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1695968029481 2023-09-29T06:13:52,437 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting cbabfb10069f48d0a50c5452ea1689e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1695968030616 2023-09-29T06:13:52,449 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/a0bfabd2be26417e9ca5684a07c473bf as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/a0bfabd2be26417e9ca5684a07c473bf 2023-09-29T06:13:52,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=62 2023-09-29T06:13:52,460 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 1668fccb65cfe5ce16ea12b0cc0df5e4/A of 1668fccb65cfe5ce16ea12b0cc0df5e4 into a0bfabd2be26417e9ca5684a07c473bf(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:52,460 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1668fccb65cfe5ce16ea12b0cc0df5e4: 2023-09-29T06:13:52,460 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4., storeName=1668fccb65cfe5ce16ea12b0cc0df5e4/A, priority=13, startTime=1695968032327; duration=0sec 2023-09-29T06:13:52,461 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:52,461 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1668fccb65cfe5ce16ea12b0cc0df5e4:A 2023-09-29T06:13:52,462 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1668fccb65cfe5ce16ea12b0cc0df5e4#C#compaction#205 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:52,463 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/93cf597f38d3482aa6e6a8df0f4546db is 50, key is test_row_0/C:col10/1695968030616/Put/seqid=0 2023-09-29T06:13:52,477 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/93cf597f38d3482aa6e6a8df0f4546db as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/93cf597f38d3482aa6e6a8df0f4546db 2023-09-29T06:13:52,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2023-09-29T06:13:52,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:52,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. as already flushing 2023-09-29T06:13:52,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:52,479 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(53): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:52,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:52,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:52,492 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 1668fccb65cfe5ce16ea12b0cc0df5e4/C of 1668fccb65cfe5ce16ea12b0cc0df5e4 into 93cf597f38d3482aa6e6a8df0f4546db(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:52,492 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1668fccb65cfe5ce16ea12b0cc0df5e4: 2023-09-29T06:13:52,492 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4., storeName=1668fccb65cfe5ce16ea12b0cc0df5e4/C, priority=13, startTime=1695968032327; duration=0sec 2023-09-29T06:13:52,493 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:52,493 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1668fccb65cfe5ce16ea12b0cc0df5e4:C 2023-09-29T06:13:52,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968092516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:52,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968092516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:52,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:52,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968092518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:52,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2023-09-29T06:13:52,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:52,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. as already flushing 2023-09-29T06:13:52,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:52,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(53): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:52,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:52,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:52,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:52,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968092718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:52,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:52,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968092719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:52,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:52,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968092721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:52,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2023-09-29T06:13:52,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:52,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. as already flushing 2023-09-29T06:13:52,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:52,790 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(53): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:52,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:52,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:52,852 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309299049f4950f624bc49cf365964506ef49_1668fccb65cfe5ce16ea12b0cc0df5e4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309299049f4950f624bc49cf365964506ef49_1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:52,854 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/d71fc11e746e446cb73312a7568f87d7, store: [table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:13:52,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/d71fc11e746e446cb73312a7568f87d7 is 175, key is test_row_0/A:col10/1695968031775/Put/seqid=0 2023-09-29T06:13:52,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2023-09-29T06:13:52,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:52,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. as already flushing 2023-09-29T06:13:52,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:52,944 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(53): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:52,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:52,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:52,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=62 2023-09-29T06:13:53,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:53,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968093021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:53,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:53,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968093022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:53,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:53,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968093023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:53,098 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2023-09-29T06:13:53,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:53,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. as already flushing 2023-09-29T06:13:53,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:53,099 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(53): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:53,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:53,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:53,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2023-09-29T06:13:53,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:53,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. as already flushing 2023-09-29T06:13:53,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:53,251 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(53): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:53,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:53,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:53,265 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=376, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/d71fc11e746e446cb73312a7568f87d7 2023-09-29T06:13:53,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/dcc89245f12842d6a16981118411b59d is 50, key is test_row_0/B:col10/1695968031775/Put/seqid=0 2023-09-29T06:13:53,404 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2023-09-29T06:13:53,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:53,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. as already flushing 2023-09-29T06:13:53,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:53,405 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(53): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:53,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:53,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:53,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:53,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968093524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:53,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:53,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968093528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:53,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:53,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968093530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:53,558 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2023-09-29T06:13:53,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:53,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. as already flushing 2023-09-29T06:13:53,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:53,558 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(53): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:53,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:53,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:53,677 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/dcc89245f12842d6a16981118411b59d 2023-09-29T06:13:53,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/f105a1e0990d4c538f61549130f9046d is 50, key is test_row_0/C:col10/1695968031775/Put/seqid=0 2023-09-29T06:13:53,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2023-09-29T06:13:53,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:53,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. as already flushing 2023-09-29T06:13:53,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:53,712 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(53): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:53,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:53,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:53,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2023-09-29T06:13:53,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:53,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. as already flushing 2023-09-29T06:13:53,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:53,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(53): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:53,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:53,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:53,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=62 2023-09-29T06:13:54,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2023-09-29T06:13:54,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:54,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. as already flushing 2023-09-29T06:13:54,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:54,018 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(53): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:54,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:54,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:54,109 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/f105a1e0990d4c538f61549130f9046d 2023-09-29T06:13:54,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/d71fc11e746e446cb73312a7568f87d7 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/d71fc11e746e446cb73312a7568f87d7 2023-09-29T06:13:54,118 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/d71fc11e746e446cb73312a7568f87d7, entries=150, sequenceid=376, filesize=30.5 K 2023-09-29T06:13:54,118 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(982): StoreScanner already closing. There is no need to updateReaders 2023-09-29T06:13:54,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/dcc89245f12842d6a16981118411b59d as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/dcc89245f12842d6a16981118411b59d 2023-09-29T06:13:54,125 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/dcc89245f12842d6a16981118411b59d, entries=150, sequenceid=376, filesize=12.0 K 2023-09-29T06:13:54,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/f105a1e0990d4c538f61549130f9046d as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/f105a1e0990d4c538f61549130f9046d 2023-09-29T06:13:54,130 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/f105a1e0990d4c538f61549130f9046d, entries=150, sequenceid=376, filesize=12.0 K 2023-09-29T06:13:54,131 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for 1668fccb65cfe5ce16ea12b0cc0df5e4 in 1737ms, sequenceid=376, compaction requested=false 2023-09-29T06:13:54,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 1668fccb65cfe5ce16ea12b0cc0df5e4: 2023-09-29T06:13:54,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2023-09-29T06:13:54,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:54,171 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2819): Flushing 1668fccb65cfe5ce16ea12b0cc0df5e4 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2023-09-29T06:13:54,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=A 2023-09-29T06:13:54,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:54,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=B 2023-09-29T06:13:54,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:54,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=C 2023-09-29T06:13:54,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:54,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929cec404f530d842179f58439b0ee1bdda_1668fccb65cfe5ce16ea12b0cc0df5e4 is 50, key is test_row_0/A:col10/1695968032413/Put/seqid=0 2023-09-29T06:13:54,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:54,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. as already flushing 2023-09-29T06:13:54,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:54,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968094553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:54,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:54,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968094554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:54,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:54,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968094555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:54,597 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929cec404f530d842179f58439b0ee1bdda_1668fccb65cfe5ce16ea12b0cc0df5e4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929cec404f530d842179f58439b0ee1bdda_1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:54,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/9bcda16378e44a1fa0eca7cfe1a637a2, store: [table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:13:54,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/9bcda16378e44a1fa0eca7cfe1a637a2 is 175, key is test_row_0/A:col10/1695968032413/Put/seqid=0 2023-09-29T06:13:54,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:54,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968094656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:54,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:54,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968094657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:54,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:54,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968094666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:54,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:54,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968094858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:54,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:54,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968094860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:54,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:54,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968094870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:55,005 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=394, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/9bcda16378e44a1fa0eca7cfe1a637a2 2023-09-29T06:13:55,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/4c8b4b192df04f5caa338eff3a01287e is 50, key is test_row_0/B:col10/1695968032413/Put/seqid=0 2023-09-29T06:13:55,023 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/4c8b4b192df04f5caa338eff3a01287e 2023-09-29T06:13:55,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/ff8b2f2554d94bc782a5f73f267569b3 is 50, key is test_row_0/C:col10/1695968032413/Put/seqid=0 2023-09-29T06:13:55,054 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/ff8b2f2554d94bc782a5f73f267569b3 2023-09-29T06:13:55,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/9bcda16378e44a1fa0eca7cfe1a637a2 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/9bcda16378e44a1fa0eca7cfe1a637a2 2023-09-29T06:13:55,073 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/9bcda16378e44a1fa0eca7cfe1a637a2, entries=150, sequenceid=394, filesize=30.5 K 2023-09-29T06:13:55,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/4c8b4b192df04f5caa338eff3a01287e as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/4c8b4b192df04f5caa338eff3a01287e 2023-09-29T06:13:55,079 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/4c8b4b192df04f5caa338eff3a01287e, entries=150, sequenceid=394, filesize=12.0 K 2023-09-29T06:13:55,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/ff8b2f2554d94bc782a5f73f267569b3 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/ff8b2f2554d94bc782a5f73f267569b3 2023-09-29T06:13:55,090 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/ff8b2f2554d94bc782a5f73f267569b3, entries=150, sequenceid=394, filesize=12.0 K 2023-09-29T06:13:55,091 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3022): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for 1668fccb65cfe5ce16ea12b0cc0df5e4 in 920ms, sequenceid=394, compaction requested=true 2023-09-29T06:13:55,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2520): Flush status journal for 1668fccb65cfe5ce16ea12b0cc0df5e4: 2023-09-29T06:13:55,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:55,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=63 2023-09-29T06:13:55,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=63 2023-09-29T06:13:55,096 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=63, resume processing ppid=62 2023-09-29T06:13:55,096 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=63, ppid=62, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2340 sec 2023-09-29T06:13:55,099 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=62, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 3.2560 sec 2023-09-29T06:13:55,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:55,167 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 1668fccb65cfe5ce16ea12b0cc0df5e4 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2023-09-29T06:13:55,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=A 2023-09-29T06:13:55,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:55,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=B 2023-09-29T06:13:55,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:55,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=C 2023-09-29T06:13:55,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:55,183 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309294ebc41e3800841dba4cd51492e9fa0fe_1668fccb65cfe5ce16ea12b0cc0df5e4 is 50, key is test_row_0/A:col10/1695968034542/Put/seqid=0 2023-09-29T06:13:55,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:55,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41454 deadline: 1695968095195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:55,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:55,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968095194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:55,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:55,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968095196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:55,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:55,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968095199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:55,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:55,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41426 deadline: 1695968095199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:55,202 DEBUG [Thread-832 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=7, retries=16, started=8190 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_0' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4., hostname=jenkins-hbase3.apache.org,39783,1695967978138, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:572) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:559) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:197) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:218) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:750) 2023-09-29T06:13:55,212 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309294ebc41e3800841dba4cd51492e9fa0fe_1668fccb65cfe5ce16ea12b0cc0df5e4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309294ebc41e3800841dba4cd51492e9fa0fe_1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:55,213 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/9a46670efa754f01ab725c07225af745, store: [table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:13:55,214 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/9a46670efa754f01ab725c07225af745 is 175, key is test_row_0/A:col10/1695968034542/Put/seqid=0 2023-09-29T06:13:55,226 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=417, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/9a46670efa754f01ab725c07225af745 2023-09-29T06:13:55,238 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/25bb783c113d43a88d7b878e3c127026 is 50, key is test_row_0/B:col10/1695968034542/Put/seqid=0 2023-09-29T06:13:55,248 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/25bb783c113d43a88d7b878e3c127026 2023-09-29T06:13:55,260 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/080c2b9daeeb4a45829a2390c521a05d is 50, key is test_row_0/C:col10/1695968034542/Put/seqid=0 2023-09-29T06:13:55,264 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/080c2b9daeeb4a45829a2390c521a05d 2023-09-29T06:13:55,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/9a46670efa754f01ab725c07225af745 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/9a46670efa754f01ab725c07225af745 2023-09-29T06:13:55,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/9a46670efa754f01ab725c07225af745, entries=200, sequenceid=417, filesize=39.0 K 2023-09-29T06:13:55,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/25bb783c113d43a88d7b878e3c127026 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/25bb783c113d43a88d7b878e3c127026 2023-09-29T06:13:55,280 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/25bb783c113d43a88d7b878e3c127026, entries=150, sequenceid=417, filesize=12.0 K 2023-09-29T06:13:55,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/080c2b9daeeb4a45829a2390c521a05d as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/080c2b9daeeb4a45829a2390c521a05d 2023-09-29T06:13:55,286 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/080c2b9daeeb4a45829a2390c521a05d, entries=150, sequenceid=417, filesize=12.0 K 2023-09-29T06:13:55,287 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 1668fccb65cfe5ce16ea12b0cc0df5e4 in 120ms, sequenceid=417, compaction requested=true 2023-09-29T06:13:55,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 1668fccb65cfe5ce16ea12b0cc0df5e4: 2023-09-29T06:13:55,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1668fccb65cfe5ce16ea12b0cc0df5e4:A, priority=-2147483648, current under compaction store size is 1 2023-09-29T06:13:55,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:55,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1668fccb65cfe5ce16ea12b0cc0df5e4:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:13:55,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:55,287 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-29T06:13:55,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1668fccb65cfe5ce16ea12b0cc0df5e4:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:13:55,287 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-29T06:13:55,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:13:55,294 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134464 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-29T06:13:55,294 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 1668fccb65cfe5ce16ea12b0cc0df5e4/A is initiating minor compaction (all files) 2023-09-29T06:13:55,295 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1668fccb65cfe5ce16ea12b0cc0df5e4/A in TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:55,295 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/a0bfabd2be26417e9ca5684a07c473bf, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/d71fc11e746e446cb73312a7568f87d7, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/9bcda16378e44a1fa0eca7cfe1a637a2, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/9a46670efa754f01ab725c07225af745] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp, totalSize=131.3 K 2023-09-29T06:13:55,295 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:55,295 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. files: [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/a0bfabd2be26417e9ca5684a07c473bf, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/d71fc11e746e446cb73312a7568f87d7, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/9bcda16378e44a1fa0eca7cfe1a637a2, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/9a46670efa754f01ab725c07225af745] 2023-09-29T06:13:55,296 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting a0bfabd2be26417e9ca5684a07c473bf, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1695968030616 2023-09-29T06:13:55,297 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-29T06:13:55,297 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 1668fccb65cfe5ce16ea12b0cc0df5e4/B is initiating minor compaction (all files) 2023-09-29T06:13:55,297 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting d71fc11e746e446cb73312a7568f87d7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1695968031775 2023-09-29T06:13:55,297 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1668fccb65cfe5ce16ea12b0cc0df5e4/B in TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:55,297 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/0a7aca788ab34c5797fdc9a787b0edb7, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/dcc89245f12842d6a16981118411b59d, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/4c8b4b192df04f5caa338eff3a01287e, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/25bb783c113d43a88d7b878e3c127026] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp, totalSize=48.8 K 2023-09-29T06:13:55,297 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 9bcda16378e44a1fa0eca7cfe1a637a2, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1695968032410 2023-09-29T06:13:55,298 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 0a7aca788ab34c5797fdc9a787b0edb7, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1695968030616 2023-09-29T06:13:55,298 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 9a46670efa754f01ab725c07225af745, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1695968034542 2023-09-29T06:13:55,298 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting dcc89245f12842d6a16981118411b59d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1695968031775 2023-09-29T06:13:55,299 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 4c8b4b192df04f5caa338eff3a01287e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1695968032410 2023-09-29T06:13:55,302 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 25bb783c113d43a88d7b878e3c127026, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1695968034542 2023-09-29T06:13:55,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:55,304 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 1668fccb65cfe5ce16ea12b0cc0df5e4 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2023-09-29T06:13:55,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=A 2023-09-29T06:13:55,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:55,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=B 2023-09-29T06:13:55,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:55,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=C 2023-09-29T06:13:55,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:55,320 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023092975b5ccec6311487f8325f08720d89842_1668fccb65cfe5ce16ea12b0cc0df5e4 is 50, key is test_row_0/A:col10/1695968035302/Put/seqid=0 2023-09-29T06:13:55,340 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1668fccb65cfe5ce16ea12b0cc0df5e4#B#compaction#215 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:55,341 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/9fa06d88873a48d4a1d94a3ff383380f is 50, key is test_row_0/B:col10/1695968034542/Put/seqid=0 2023-09-29T06:13:55,347 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:13:55,358 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202309295b40aebdee0b4e438573f43d2f72e60c_1668fccb65cfe5ce16ea12b0cc0df5e4 store=[table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:13:55,360 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202309295b40aebdee0b4e438573f43d2f72e60c_1668fccb65cfe5ce16ea12b0cc0df5e4, store=[table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:13:55,360 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309295b40aebdee0b4e438573f43d2f72e60c_1668fccb65cfe5ce16ea12b0cc0df5e4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:13:55,373 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023092975b5ccec6311487f8325f08720d89842_1668fccb65cfe5ce16ea12b0cc0df5e4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023092975b5ccec6311487f8325f08720d89842_1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:55,374 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/c30095dd60e144c29cc8d32b17789be3, store: [table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:13:55,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/c30095dd60e144c29cc8d32b17789be3 is 175, key is test_row_0/A:col10/1695968035302/Put/seqid=0 2023-09-29T06:13:55,380 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/9fa06d88873a48d4a1d94a3ff383380f as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/9fa06d88873a48d4a1d94a3ff383380f 2023-09-29T06:13:55,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4509) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4440) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:4929) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:4923) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:4919) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3215) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:55,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968095338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:55,385 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 1668fccb65cfe5ce16ea12b0cc0df5e4/B of 1668fccb65cfe5ce16ea12b0cc0df5e4 into 9fa06d88873a48d4a1d94a3ff383380f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:55,385 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1668fccb65cfe5ce16ea12b0cc0df5e4: 2023-09-29T06:13:55,385 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4., storeName=1668fccb65cfe5ce16ea12b0cc0df5e4/B, priority=12, startTime=1695968035287; duration=0sec 2023-09-29T06:13:55,385 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:13:55,385 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1668fccb65cfe5ce16ea12b0cc0df5e4:B 2023-09-29T06:13:55,385 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-29T06:13:55,387 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=433, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/c30095dd60e144c29cc8d32b17789be3 2023-09-29T06:13:55,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:55,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968095386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:55,389 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-29T06:13:55,389 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 1668fccb65cfe5ce16ea12b0cc0df5e4/C is initiating minor compaction (all files) 2023-09-29T06:13:55,389 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1668fccb65cfe5ce16ea12b0cc0df5e4/C in TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:55,389 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/93cf597f38d3482aa6e6a8df0f4546db, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/f105a1e0990d4c538f61549130f9046d, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/ff8b2f2554d94bc782a5f73f267569b3, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/080c2b9daeeb4a45829a2390c521a05d] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp, totalSize=48.8 K 2023-09-29T06:13:55,389 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 93cf597f38d3482aa6e6a8df0f4546db, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1695968030616 2023-09-29T06:13:55,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:55,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968095389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:55,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:55,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41454 deadline: 1695968095389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:55,394 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting f105a1e0990d4c538f61549130f9046d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1695968031775 2023-09-29T06:13:55,395 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting ff8b2f2554d94bc782a5f73f267569b3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1695968032410 2023-09-29T06:13:55,395 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 080c2b9daeeb4a45829a2390c521a05d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1695968034542 2023-09-29T06:13:55,407 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/66f8914a3047450ab9be270684153626 is 50, key is test_row_0/B:col10/1695968035302/Put/seqid=0 2023-09-29T06:13:55,415 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1668fccb65cfe5ce16ea12b0cc0df5e4#C#compaction#218 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:55,416 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/7d9d888ea0844f588275d0e1791499ae is 50, key is test_row_0/C:col10/1695968034542/Put/seqid=0 2023-09-29T06:13:55,431 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/66f8914a3047450ab9be270684153626 2023-09-29T06:13:55,437 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/7d9d888ea0844f588275d0e1791499ae as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/7d9d888ea0844f588275d0e1791499ae 2023-09-29T06:13:55,444 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/336f3cd52546455ebc0cb6bd2db2b949 is 50, key is test_row_0/C:col10/1695968035302/Put/seqid=0 2023-09-29T06:13:55,448 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 1668fccb65cfe5ce16ea12b0cc0df5e4/C of 1668fccb65cfe5ce16ea12b0cc0df5e4 into 7d9d888ea0844f588275d0e1791499ae(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:55,448 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1668fccb65cfe5ce16ea12b0cc0df5e4: 2023-09-29T06:13:55,448 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4., storeName=1668fccb65cfe5ce16ea12b0cc0df5e4/C, priority=12, startTime=1695968035287; duration=0sec 2023-09-29T06:13:55,448 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:55,448 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1668fccb65cfe5ce16ea12b0cc0df5e4:C 2023-09-29T06:13:55,479 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/336f3cd52546455ebc0cb6bd2db2b949 2023-09-29T06:13:55,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/c30095dd60e144c29cc8d32b17789be3 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/c30095dd60e144c29cc8d32b17789be3 2023-09-29T06:13:55,490 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/c30095dd60e144c29cc8d32b17789be3, entries=250, sequenceid=433, filesize=47.5 K 2023-09-29T06:13:55,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/66f8914a3047450ab9be270684153626 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/66f8914a3047450ab9be270684153626 2023-09-29T06:13:55,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:55,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968095487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:55,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:55,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968095489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:55,495 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/66f8914a3047450ab9be270684153626, entries=150, sequenceid=433, filesize=12.0 K 2023-09-29T06:13:55,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:55,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:55,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41454 deadline: 1695968095494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:55,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968095494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:55,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/336f3cd52546455ebc0cb6bd2db2b949 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/336f3cd52546455ebc0cb6bd2db2b949 2023-09-29T06:13:55,500 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/336f3cd52546455ebc0cb6bd2db2b949, entries=150, sequenceid=433, filesize=12.0 K 2023-09-29T06:13:55,501 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for 1668fccb65cfe5ce16ea12b0cc0df5e4 in 197ms, sequenceid=433, compaction requested=false 2023-09-29T06:13:55,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 1668fccb65cfe5ce16ea12b0cc0df5e4: 2023-09-29T06:13:55,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:55,696 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 1668fccb65cfe5ce16ea12b0cc0df5e4 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2023-09-29T06:13:55,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=A 2023-09-29T06:13:55,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:55,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=B 2023-09-29T06:13:55,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:55,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=C 2023-09-29T06:13:55,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:55,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929f4c12b9649ae45299330baa06355c0c1_1668fccb65cfe5ce16ea12b0cc0df5e4 is 50, key is test_row_0/A:col10/1695968035695/Put/seqid=0 2023-09-29T06:13:55,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:55,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968095709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:55,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:55,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41454 deadline: 1695968095710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:55,713 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929f4c12b9649ae45299330baa06355c0c1_1668fccb65cfe5ce16ea12b0cc0df5e4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929f4c12b9649ae45299330baa06355c0c1_1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:55,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:55,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968095711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:55,714 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/fb8d78a75bbf4a4b9c78f142b404277d, store: [table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:13:55,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:55,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968095712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:55,715 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/fb8d78a75bbf4a4b9c78f142b404277d is 175, key is test_row_0/A:col10/1695968035695/Put/seqid=0 2023-09-29T06:13:55,785 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1668fccb65cfe5ce16ea12b0cc0df5e4#A#compaction#216 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:55,785 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/4f84450731d14b1aaa5da960fd95b37e is 175, key is test_row_0/A:col10/1695968034542/Put/seqid=0 2023-09-29T06:13:55,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:55,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968095813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:55,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:55,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41454 deadline: 1695968095814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:55,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:55,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968095815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:55,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:55,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968095816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:55,823 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/4f84450731d14b1aaa5da960fd95b37e as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/4f84450731d14b1aaa5da960fd95b37e 2023-09-29T06:13:55,830 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 1668fccb65cfe5ce16ea12b0cc0df5e4/A of 1668fccb65cfe5ce16ea12b0cc0df5e4 into 4f84450731d14b1aaa5da960fd95b37e(size=31.4 K), total size for store is 78.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:55,830 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1668fccb65cfe5ce16ea12b0cc0df5e4: 2023-09-29T06:13:55,830 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4., storeName=1668fccb65cfe5ce16ea12b0cc0df5e4/A, priority=12, startTime=1695968035287; duration=0sec 2023-09-29T06:13:55,831 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:55,831 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1668fccb65cfe5ce16ea12b0cc0df5e4:A 2023-09-29T06:13:55,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=62 2023-09-29T06:13:55,953 INFO [Thread-842 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 62 completed 2023-09-29T06:13:55,955 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:13:55,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2023-09-29T06:13:55,960 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:13:55,961 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:13:55,961 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:13:55,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=64 2023-09-29T06:13:55,997 DEBUG [Thread-845 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x59bbfd07 to 127.0.0.1:49344 2023-09-29T06:13:55,997 DEBUG [Thread-845 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:13:55,998 DEBUG [Thread-849 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x6583e14a to 127.0.0.1:49344 2023-09-29T06:13:55,998 DEBUG [Thread-849 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:13:55,999 DEBUG [Thread-843 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x1b138f3c to 127.0.0.1:49344 2023-09-29T06:13:55,999 DEBUG [Thread-843 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:13:55,999 DEBUG [Thread-847 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0e69549a to 127.0.0.1:49344 2023-09-29T06:13:55,999 DEBUG [Thread-847 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:13:56,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:56,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:56,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968096017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:56,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41454 deadline: 1695968096016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:56,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:56,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968096018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:56,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:56,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968096021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:56,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=64 2023-09-29T06:13:56,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2023-09-29T06:13:56,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:56,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. as already flushing 2023-09-29T06:13:56,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:56,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(53): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:56,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:56,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:56,129 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=458, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/fb8d78a75bbf4a4b9c78f142b404277d 2023-09-29T06:13:56,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/6c60e803bbcc4799bbf8f3a31595a3d8 is 50, key is test_row_0/B:col10/1695968035695/Put/seqid=0 2023-09-29T06:13:56,149 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/6c60e803bbcc4799bbf8f3a31595a3d8 2023-09-29T06:13:56,157 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/65f9f33e4bdc477681ca2d88db590655 is 50, key is test_row_0/C:col10/1695968035695/Put/seqid=0 2023-09-29T06:13:56,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=64 2023-09-29T06:13:56,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2023-09-29T06:13:56,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:56,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. as already flushing 2023-09-29T06:13:56,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:56,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(53): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:56,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:56,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:56,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:56,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41454 deadline: 1695968096318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:56,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:56,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41440 deadline: 1695968096319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:56,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:56,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41494 deadline: 1695968096320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:56,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:13:56,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41458 deadline: 1695968096323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:13:56,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2023-09-29T06:13:56,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:56,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. as already flushing 2023-09-29T06:13:56,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:56,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(53): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:56,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:56,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:56,562 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/65f9f33e4bdc477681ca2d88db590655 2023-09-29T06:13:56,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=64 2023-09-29T06:13:56,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/fb8d78a75bbf4a4b9c78f142b404277d as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/fb8d78a75bbf4a4b9c78f142b404277d 2023-09-29T06:13:56,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/fb8d78a75bbf4a4b9c78f142b404277d, entries=200, sequenceid=458, filesize=39.0 K 2023-09-29T06:13:56,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/6c60e803bbcc4799bbf8f3a31595a3d8 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/6c60e803bbcc4799bbf8f3a31595a3d8 2023-09-29T06:13:56,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2023-09-29T06:13:56,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:56,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. as already flushing 2023-09-29T06:13:56,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:56,577 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(53): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:56,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:56,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:13:56,579 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/6c60e803bbcc4799bbf8f3a31595a3d8, entries=150, sequenceid=458, filesize=12.0 K 2023-09-29T06:13:56,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/65f9f33e4bdc477681ca2d88db590655 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/65f9f33e4bdc477681ca2d88db590655 2023-09-29T06:13:56,585 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/65f9f33e4bdc477681ca2d88db590655, entries=150, sequenceid=458, filesize=12.0 K 2023-09-29T06:13:56,586 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 1668fccb65cfe5ce16ea12b0cc0df5e4 in 891ms, sequenceid=458, compaction requested=true 2023-09-29T06:13:56,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 1668fccb65cfe5ce16ea12b0cc0df5e4: 2023-09-29T06:13:56,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1668fccb65cfe5ce16ea12b0cc0df5e4:A, priority=-2147483648, current under compaction store size is 1 2023-09-29T06:13:56,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:56,586 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:13:56,586 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:13:56,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1668fccb65cfe5ce16ea12b0cc0df5e4:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:13:56,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:56,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1668fccb65cfe5ce16ea12b0cc0df5e4:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:13:56,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:13:56,587 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 120746 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:13:56,587 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 1668fccb65cfe5ce16ea12b0cc0df5e4/A is initiating minor compaction (all files) 2023-09-29T06:13:56,587 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37806 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:13:56,587 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1668fccb65cfe5ce16ea12b0cc0df5e4/A in TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:56,587 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 1668fccb65cfe5ce16ea12b0cc0df5e4/B is initiating minor compaction (all files) 2023-09-29T06:13:56,587 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/4f84450731d14b1aaa5da960fd95b37e, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/c30095dd60e144c29cc8d32b17789be3, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/fb8d78a75bbf4a4b9c78f142b404277d] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp, totalSize=117.9 K 2023-09-29T06:13:56,587 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:56,587 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1668fccb65cfe5ce16ea12b0cc0df5e4/B in TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:56,588 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. files: [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/4f84450731d14b1aaa5da960fd95b37e, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/c30095dd60e144c29cc8d32b17789be3, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/fb8d78a75bbf4a4b9c78f142b404277d] 2023-09-29T06:13:56,588 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/9fa06d88873a48d4a1d94a3ff383380f, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/66f8914a3047450ab9be270684153626, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/6c60e803bbcc4799bbf8f3a31595a3d8] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp, totalSize=36.9 K 2023-09-29T06:13:56,588 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 4f84450731d14b1aaa5da960fd95b37e, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1695968034542 2023-09-29T06:13:56,588 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 9fa06d88873a48d4a1d94a3ff383380f, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1695968034542 2023-09-29T06:13:56,588 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting c30095dd60e144c29cc8d32b17789be3, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1695968035187 2023-09-29T06:13:56,588 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 66f8914a3047450ab9be270684153626, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1695968035197 2023-09-29T06:13:56,589 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting fb8d78a75bbf4a4b9c78f142b404277d, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1695968035344 2023-09-29T06:13:56,589 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 6c60e803bbcc4799bbf8f3a31595a3d8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1695968035344 2023-09-29T06:13:56,602 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1668fccb65cfe5ce16ea12b0cc0df5e4#B#compaction#223 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:56,602 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/f8435501d558436da342d1c174894b8c is 50, key is test_row_0/B:col10/1695968035695/Put/seqid=0 2023-09-29T06:13:56,610 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:13:56,617 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202309297ca4d90ed03849eaae64e0efeaf863d1_1668fccb65cfe5ce16ea12b0cc0df5e4 store=[table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:13:56,637 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202309297ca4d90ed03849eaae64e0efeaf863d1_1668fccb65cfe5ce16ea12b0cc0df5e4, store=[table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:13:56,638 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309297ca4d90ed03849eaae64e0efeaf863d1_1668fccb65cfe5ce16ea12b0cc0df5e4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:13:56,650 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1668fccb65cfe5ce16ea12b0cc0df5e4#A#compaction#224 average throughput is 0.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:56,650 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/6d7664195c4c43a2b04a1d71fc644836 is 175, key is test_row_0/A:col10/1695968035695/Put/seqid=0 2023-09-29T06:13:56,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2023-09-29T06:13:56,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:56,730 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2819): Flushing 1668fccb65cfe5ce16ea12b0cc0df5e4 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2023-09-29T06:13:56,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=A 2023-09-29T06:13:56,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:56,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=B 2023-09-29T06:13:56,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:56,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=C 2023-09-29T06:13:56,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:13:56,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929205ef05d47f64cf6ab0467c191341b4f_1668fccb65cfe5ce16ea12b0cc0df5e4 is 50, key is test_row_0/A:col10/1695968035711/Put/seqid=0 2023-09-29T06:13:56,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:56,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. as already flushing 2023-09-29T06:13:56,823 DEBUG [Thread-834 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x4364a9e3 to 127.0.0.1:49344 2023-09-29T06:13:56,823 DEBUG [Thread-834 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:13:56,823 DEBUG [Thread-838 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0d682546 to 127.0.0.1:49344 2023-09-29T06:13:56,823 DEBUG [Thread-838 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:13:56,826 DEBUG [Thread-840 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x20caeb3b to 127.0.0.1:49344 2023-09-29T06:13:56,826 DEBUG [Thread-836 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x55a8701b to 127.0.0.1:49344 2023-09-29T06:13:56,826 DEBUG [Thread-840 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:13:56,826 DEBUG [Thread-836 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:13:57,037 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/f8435501d558436da342d1c174894b8c as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/f8435501d558436da342d1c174894b8c 2023-09-29T06:13:57,043 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 1668fccb65cfe5ce16ea12b0cc0df5e4/B of 1668fccb65cfe5ce16ea12b0cc0df5e4 into f8435501d558436da342d1c174894b8c(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:57,043 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1668fccb65cfe5ce16ea12b0cc0df5e4: 2023-09-29T06:13:57,043 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4., storeName=1668fccb65cfe5ce16ea12b0cc0df5e4/B, priority=13, startTime=1695968036586; duration=0sec 2023-09-29T06:13:57,043 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:13:57,043 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1668fccb65cfe5ce16ea12b0cc0df5e4:B 2023-09-29T06:13:57,043 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:13:57,048 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37806 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:13:57,048 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 1668fccb65cfe5ce16ea12b0cc0df5e4/C is initiating minor compaction (all files) 2023-09-29T06:13:57,049 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 1668fccb65cfe5ce16ea12b0cc0df5e4/C in TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:57,049 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/7d9d888ea0844f588275d0e1791499ae, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/336f3cd52546455ebc0cb6bd2db2b949, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/65f9f33e4bdc477681ca2d88db590655] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp, totalSize=36.9 K 2023-09-29T06:13:57,049 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 7d9d888ea0844f588275d0e1791499ae, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1695968034542 2023-09-29T06:13:57,049 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 336f3cd52546455ebc0cb6bd2db2b949, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1695968035197 2023-09-29T06:13:57,050 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 65f9f33e4bdc477681ca2d88db590655, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1695968035344 2023-09-29T06:13:57,057 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1668fccb65cfe5ce16ea12b0cc0df5e4#C#compaction#226 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:13:57,058 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/d6dc7f54197649f9b51f61b2fe1f4a53 is 50, key is test_row_0/C:col10/1695968035695/Put/seqid=0 2023-09-29T06:13:57,065 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/6d7664195c4c43a2b04a1d71fc644836 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/6d7664195c4c43a2b04a1d71fc644836 2023-09-29T06:13:57,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=64 2023-09-29T06:13:57,068 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/d6dc7f54197649f9b51f61b2fe1f4a53 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/d6dc7f54197649f9b51f61b2fe1f4a53 2023-09-29T06:13:57,073 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 1668fccb65cfe5ce16ea12b0cc0df5e4/C of 1668fccb65cfe5ce16ea12b0cc0df5e4 into d6dc7f54197649f9b51f61b2fe1f4a53(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:57,073 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1668fccb65cfe5ce16ea12b0cc0df5e4: 2023-09-29T06:13:57,073 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4., storeName=1668fccb65cfe5ce16ea12b0cc0df5e4/C, priority=13, startTime=1695968036586; duration=0sec 2023-09-29T06:13:57,073 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:57,073 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1668fccb65cfe5ce16ea12b0cc0df5e4:C 2023-09-29T06:13:57,076 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 1668fccb65cfe5ce16ea12b0cc0df5e4/A of 1668fccb65cfe5ce16ea12b0cc0df5e4 into 6d7664195c4c43a2b04a1d71fc644836(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:13:57,076 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 1668fccb65cfe5ce16ea12b0cc0df5e4: 2023-09-29T06:13:57,076 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4., storeName=1668fccb65cfe5ce16ea12b0cc0df5e4/A, priority=13, startTime=1695968036586; duration=0sec 2023-09-29T06:13:57,076 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:13:57,076 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1668fccb65cfe5ce16ea12b0cc0df5e4:A 2023-09-29T06:13:57,152 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929205ef05d47f64cf6ab0467c191341b4f_1668fccb65cfe5ce16ea12b0cc0df5e4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929205ef05d47f64cf6ab0467c191341b4f_1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:13:57,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/238a584958b5430fabea55e251f39c31, store: [table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:13:57,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/238a584958b5430fabea55e251f39c31 is 175, key is test_row_0/A:col10/1695968035711/Put/seqid=0 2023-09-29T06:13:57,159 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=472, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/238a584958b5430fabea55e251f39c31 2023-09-29T06:13:57,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/e44785e0ad4d401fa77dbe77f7c4ad59 is 50, key is test_row_0/B:col10/1695968035711/Put/seqid=0 2023-09-29T06:13:57,570 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/e44785e0ad4d401fa77dbe77f7c4ad59 2023-09-29T06:13:57,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/10e2b799a60842b6a52ca5ccba0c6a7f is 50, key is test_row_0/C:col10/1695968035711/Put/seqid=0 2023-09-29T06:13:57,582 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/10e2b799a60842b6a52ca5ccba0c6a7f 2023-09-29T06:13:57,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/238a584958b5430fabea55e251f39c31 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/238a584958b5430fabea55e251f39c31 2023-09-29T06:13:57,592 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/238a584958b5430fabea55e251f39c31, entries=150, sequenceid=472, filesize=30.5 K 2023-09-29T06:13:57,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/e44785e0ad4d401fa77dbe77f7c4ad59 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/e44785e0ad4d401fa77dbe77f7c4ad59 2023-09-29T06:13:57,599 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/e44785e0ad4d401fa77dbe77f7c4ad59, entries=150, sequenceid=472, filesize=12.0 K 2023-09-29T06:13:57,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/10e2b799a60842b6a52ca5ccba0c6a7f as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/10e2b799a60842b6a52ca5ccba0c6a7f 2023-09-29T06:13:57,605 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/10e2b799a60842b6a52ca5ccba0c6a7f, entries=150, sequenceid=472, filesize=12.0 K 2023-09-29T06:13:57,606 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3022): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=26.84 KB/27480 for 1668fccb65cfe5ce16ea12b0cc0df5e4 in 876ms, sequenceid=472, compaction requested=false 2023-09-29T06:13:57,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2520): Flush status journal for 1668fccb65cfe5ce16ea12b0cc0df5e4: 2023-09-29T06:13:57,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:13:57,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=65 2023-09-29T06:13:57,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=65 2023-09-29T06:13:57,608 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=65, resume processing ppid=64 2023-09-29T06:13:57,608 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=65, ppid=64, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6460 sec 2023-09-29T06:13:57,611 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=64, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.6540 sec 2023-09-29T06:13:58,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=64 2023-09-29T06:13:58,070 INFO [Thread-842 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 64 completed 2023-09-29T06:14:01,964 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 013609be28cb78869a9efb7623dcb4f7 changed from -1.0 to 0.0, refreshing cache 2023-09-29T06:14:05,213 DEBUG [Thread-832 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x41c47c71 to 127.0.0.1:49344 2023-09-29T06:14:05,213 DEBUG [Thread-832 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:14:05,213 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2023-09-29T06:14:05,213 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2023-09-29T06:14:05,213 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2023-09-29T06:14:05,213 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(394): wrote 92 2023-09-29T06:14:05,213 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(394): wrote 94 2023-09-29T06:14:05,213 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2023-09-29T06:14:05,213 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(396): Readers: 2023-09-29T06:14:05,214 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(398): read 5631 2023-09-29T06:14:05,214 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(398): read 5275 2023-09-29T06:14:05,214 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2023-09-29T06:14:05,214 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(402): scanned 2459 2023-09-29T06:14:05,214 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(403): verified 7377 rows 2023-09-29T06:14:05,214 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(402): scanned 2474 2023-09-29T06:14:05,214 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(403): verified 7422 rows 2023-09-29T06:14:05,214 INFO [Listener at localhost/42843 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-09-29T06:14:05,214 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x1394947b to 127.0.0.1:49344 2023-09-29T06:14:05,214 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:14:05,217 INFO [Listener at localhost/42843 {}] client.HBaseAdmin$18(967): Started disable of TestAcidGuarantees 2023-09-29T06:14:05,219 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$12(2664): Client=jenkins//172.31.12.81 disable TestAcidGuarantees 2023-09-29T06:14:05,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=66, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=TestAcidGuarantees 2023-09-29T06:14:05,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=66 2023-09-29T06:14:05,224 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968045224"}]},"ts":"1695968045224"} 2023-09-29T06:14:05,225 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2023-09-29T06:14:05,228 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(293): Set TestAcidGuarantees to state=DISABLING 2023-09-29T06:14:05,229 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=1668fccb65cfe5ce16ea12b0cc0df5e4, UNASSIGN}] 2023-09-29T06:14:05,230 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=67, ppid=66, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=1668fccb65cfe5ce16ea12b0cc0df5e4, UNASSIGN 2023-09-29T06:14:05,231 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=67 updating hbase:meta row=1668fccb65cfe5ce16ea12b0cc0df5e4, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:05,232 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=68, ppid=67, state=RUNNABLE; CloseRegionProcedure 1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138}] 2023-09-29T06:14:05,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=66 2023-09-29T06:14:05,384 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] handler.UnassignRegionHandler(119): Close 1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:14:05,384 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegion(1668): Closing 1668fccb65cfe5ce16ea12b0cc0df5e4, disabling compactions & flushes 2023-09-29T06:14:05,384 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegion(1690): Closing region TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:14:05,384 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegion(1711): Waiting without time limit for close lock on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:14:05,384 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegion(1778): Acquired close lock on TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. after waiting 0 ms 2023-09-29T06:14:05,384 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegion(1788): Updates disabled for region TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:14:05,384 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegion(2819): Flushing 1668fccb65cfe5ce16ea12b0cc0df5e4 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2023-09-29T06:14:05,385 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=A 2023-09-29T06:14:05,385 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:05,385 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=B 2023-09-29T06:14:05,385 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:05,385 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 1668fccb65cfe5ce16ea12b0cc0df5e4, store=C 2023-09-29T06:14:05,385 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:05,393 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929b0b6620a0178413bbfe437e3a2114976_1668fccb65cfe5ce16ea12b0cc0df5e4 is 50, key is test_row_0/A:col10/1695968045211/Put/seqid=0 2023-09-29T06:14:05,411 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929b0b6620a0178413bbfe437e3a2114976_1668fccb65cfe5ce16ea12b0cc0df5e4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929b0b6620a0178413bbfe437e3a2114976_1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:14:05,413 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/b1c1986e04dc431cb284332c5b74e868, store: [table=TestAcidGuarantees family=A region=1668fccb65cfe5ce16ea12b0cc0df5e4] 2023-09-29T06:14:05,414 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/b1c1986e04dc431cb284332c5b74e868 is 175, key is test_row_0/A:col10/1695968045211/Put/seqid=0 2023-09-29T06:14:05,428 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=483, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/b1c1986e04dc431cb284332c5b74e868 2023-09-29T06:14:05,436 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/31d7fa1fd7234ed1955fbf96a012a7dc is 50, key is test_row_0/B:col10/1695968045211/Put/seqid=0 2023-09-29T06:14:05,443 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/31d7fa1fd7234ed1955fbf96a012a7dc 2023-09-29T06:14:05,450 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/9b807447882f49faa0c96d8f9184be4c is 50, key is test_row_0/C:col10/1695968045211/Put/seqid=0 2023-09-29T06:14:05,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=66 2023-09-29T06:14:05,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=66 2023-09-29T06:14:05,855 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/9b807447882f49faa0c96d8f9184be4c 2023-09-29T06:14:05,860 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/A/b1c1986e04dc431cb284332c5b74e868 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/b1c1986e04dc431cb284332c5b74e868 2023-09-29T06:14:05,864 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/b1c1986e04dc431cb284332c5b74e868, entries=150, sequenceid=483, filesize=30.5 K 2023-09-29T06:14:05,865 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/B/31d7fa1fd7234ed1955fbf96a012a7dc as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/31d7fa1fd7234ed1955fbf96a012a7dc 2023-09-29T06:14:05,870 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/31d7fa1fd7234ed1955fbf96a012a7dc, entries=150, sequenceid=483, filesize=12.0 K 2023-09-29T06:14:05,871 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/.tmp/C/9b807447882f49faa0c96d8f9184be4c as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/9b807447882f49faa0c96d8f9184be4c 2023-09-29T06:14:05,876 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/9b807447882f49faa0c96d8f9184be4c, entries=150, sequenceid=483, filesize=12.0 K 2023-09-29T06:14:05,877 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegion(3022): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 1668fccb65cfe5ce16ea12b0cc0df5e4 in 493ms, sequenceid=483, compaction requested=true 2023-09-29T06:14:05,903 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/99c7d22c33b94115a55e0f7fcd53e5fc, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/96698babbf764e2ea94216be33f16da5, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/fad17e62f93a488cad0e9cb91ab3f9b6, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/690ebdb406554486af399eb64e9e6caf, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/20bb16d93b454799a89ff4f79dcb4006, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/40325d109222441a83e5e6f4d54aead8, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/e4c7e2d52d1f428fadcfedd37feeea02, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/b1f19f332dad415687a1955cc6cfecab, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/6ee86473ab2343a788657ec685706263, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/ca9d61755642409ca623a485af340e44, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/9d1fb5935ab9487dbbb5940409e5be07, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/ce362edc2c30416abd3fcc2a8538dce4, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/5518b8580fb84fac9a6870dc6516b3e4, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/02d8dbe4b2dd479695f2e5afc576b206, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/c3e7fbfdb7744f688b3df03af100bb90, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/c6a2d88f61434d58ae80b0d4093fac9d, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/e513875edf6f4d7bb6c8f09de557474d, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/f1a4b3fe2d454dfbbf312ed533c92654, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/5d1decb820d9406dbf9e60cdb2c05399, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/39a5ed35e50245d494641c032d3bd216, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/be6a80eb693a4722840efe2720a394e9, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/2d64ebd9fd554a2782b9d17e6eb836d8, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/7e274237aa6d4577be8b839cfdd64604, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/213cc87396074545a7ce164788e9e9b7, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/a0bfabd2be26417e9ca5684a07c473bf, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/d71fc11e746e446cb73312a7568f87d7, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/9bcda16378e44a1fa0eca7cfe1a637a2, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/9a46670efa754f01ab725c07225af745, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/4f84450731d14b1aaa5da960fd95b37e, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/c30095dd60e144c29cc8d32b17789be3, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/fb8d78a75bbf4a4b9c78f142b404277d] to archive 2023-09-29T06:14:05,905 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2023-09-29T06:14:05,907 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/99c7d22c33b94115a55e0f7fcd53e5fc to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/99c7d22c33b94115a55e0f7fcd53e5fc 2023-09-29T06:14:05,909 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/96698babbf764e2ea94216be33f16da5 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/96698babbf764e2ea94216be33f16da5 2023-09-29T06:14:05,910 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/fad17e62f93a488cad0e9cb91ab3f9b6 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/fad17e62f93a488cad0e9cb91ab3f9b6 2023-09-29T06:14:05,912 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/690ebdb406554486af399eb64e9e6caf to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/690ebdb406554486af399eb64e9e6caf 2023-09-29T06:14:05,913 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/20bb16d93b454799a89ff4f79dcb4006 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/20bb16d93b454799a89ff4f79dcb4006 2023-09-29T06:14:05,914 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/40325d109222441a83e5e6f4d54aead8 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/40325d109222441a83e5e6f4d54aead8 2023-09-29T06:14:05,916 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/e4c7e2d52d1f428fadcfedd37feeea02 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/e4c7e2d52d1f428fadcfedd37feeea02 2023-09-29T06:14:05,917 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/b1f19f332dad415687a1955cc6cfecab to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/b1f19f332dad415687a1955cc6cfecab 2023-09-29T06:14:05,919 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/6ee86473ab2343a788657ec685706263 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/6ee86473ab2343a788657ec685706263 2023-09-29T06:14:05,920 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/ca9d61755642409ca623a485af340e44 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/ca9d61755642409ca623a485af340e44 2023-09-29T06:14:05,921 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/9d1fb5935ab9487dbbb5940409e5be07 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/9d1fb5935ab9487dbbb5940409e5be07 2023-09-29T06:14:05,922 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/ce362edc2c30416abd3fcc2a8538dce4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/ce362edc2c30416abd3fcc2a8538dce4 2023-09-29T06:14:05,928 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/5518b8580fb84fac9a6870dc6516b3e4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/5518b8580fb84fac9a6870dc6516b3e4 2023-09-29T06:14:05,929 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/02d8dbe4b2dd479695f2e5afc576b206 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/02d8dbe4b2dd479695f2e5afc576b206 2023-09-29T06:14:05,930 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/c3e7fbfdb7744f688b3df03af100bb90 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/c3e7fbfdb7744f688b3df03af100bb90 2023-09-29T06:14:05,931 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/c6a2d88f61434d58ae80b0d4093fac9d to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/c6a2d88f61434d58ae80b0d4093fac9d 2023-09-29T06:14:05,932 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/e513875edf6f4d7bb6c8f09de557474d to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/e513875edf6f4d7bb6c8f09de557474d 2023-09-29T06:14:05,933 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/f1a4b3fe2d454dfbbf312ed533c92654 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/f1a4b3fe2d454dfbbf312ed533c92654 2023-09-29T06:14:05,934 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/5d1decb820d9406dbf9e60cdb2c05399 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/5d1decb820d9406dbf9e60cdb2c05399 2023-09-29T06:14:05,935 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/39a5ed35e50245d494641c032d3bd216 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/39a5ed35e50245d494641c032d3bd216 2023-09-29T06:14:05,936 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/be6a80eb693a4722840efe2720a394e9 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/be6a80eb693a4722840efe2720a394e9 2023-09-29T06:14:05,936 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/2d64ebd9fd554a2782b9d17e6eb836d8 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/2d64ebd9fd554a2782b9d17e6eb836d8 2023-09-29T06:14:05,937 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/7e274237aa6d4577be8b839cfdd64604 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/7e274237aa6d4577be8b839cfdd64604 2023-09-29T06:14:05,938 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/213cc87396074545a7ce164788e9e9b7 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/213cc87396074545a7ce164788e9e9b7 2023-09-29T06:14:05,940 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/a0bfabd2be26417e9ca5684a07c473bf to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/a0bfabd2be26417e9ca5684a07c473bf 2023-09-29T06:14:05,941 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/d71fc11e746e446cb73312a7568f87d7 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/d71fc11e746e446cb73312a7568f87d7 2023-09-29T06:14:05,942 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/9bcda16378e44a1fa0eca7cfe1a637a2 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/9bcda16378e44a1fa0eca7cfe1a637a2 2023-09-29T06:14:05,943 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/9a46670efa754f01ab725c07225af745 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/9a46670efa754f01ab725c07225af745 2023-09-29T06:14:05,944 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/4f84450731d14b1aaa5da960fd95b37e to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/4f84450731d14b1aaa5da960fd95b37e 2023-09-29T06:14:05,946 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/c30095dd60e144c29cc8d32b17789be3 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/c30095dd60e144c29cc8d32b17789be3 2023-09-29T06:14:05,949 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/fb8d78a75bbf4a4b9c78f142b404277d to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/fb8d78a75bbf4a4b9c78f142b404277d 2023-09-29T06:14:05,952 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/f812d3ee16bb479cb4faaf3b751a3c52, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/b3370aaff89d428c8d24eff9bd51c8eb, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/a7408cfa16174a08a43d1d2856392b19, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/dc75ed81e7274a4c8fc9d30ae3c50556, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/39aba1f938054a6e957bf82487842817, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/a2a0e20e60674e0aa39d7d8dbf9f007d, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/a1356accbe344dd198ead53dd1da6bed, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/7b23583faefa45929d0e787c3296c92e, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/e9bd2b6eb08e4e62bc101f48c1636b8a, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/8d0b29d3d0b143aaa452583c001ce668, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/4b4ce7d6d60b46b99698f597cb331aad, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/2890bf40263b4b7e86e9bb811e28ff4b, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/b695634ad3fb4642ab75deaf324386c8, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/6712827553864291b6437d5cabffb9d1, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/3e0e2153753440468db40372d268bdee, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/d1c6ff4fc86d4fcfa94152d97db47d2a, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/841bf539ffad45d28bad0935f07d5ef1, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/fb4554ca49274424a35c1a468afc0c32, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/8e897e1875974cd59bab8c4abf91f6e5, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/2ca492b23399499abf9424f420d19b75, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/dce408cf71f74e999c731d8d8d54847c, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/34bd0b053b8a4f468f1fa1773f49b285, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/2da76a3ea0c44799bfa6bc70d6e509d0, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/0a7aca788ab34c5797fdc9a787b0edb7, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/ea9180622cf042cda8999f5de5e0cf97, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/dcc89245f12842d6a16981118411b59d, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/4c8b4b192df04f5caa338eff3a01287e, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/9fa06d88873a48d4a1d94a3ff383380f, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/25bb783c113d43a88d7b878e3c127026, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/66f8914a3047450ab9be270684153626, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/6c60e803bbcc4799bbf8f3a31595a3d8] to archive 2023-09-29T06:14:05,952 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2023-09-29T06:14:05,954 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/f812d3ee16bb479cb4faaf3b751a3c52 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/f812d3ee16bb479cb4faaf3b751a3c52 2023-09-29T06:14:05,956 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/b3370aaff89d428c8d24eff9bd51c8eb to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/b3370aaff89d428c8d24eff9bd51c8eb 2023-09-29T06:14:05,957 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/a7408cfa16174a08a43d1d2856392b19 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/a7408cfa16174a08a43d1d2856392b19 2023-09-29T06:14:05,958 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/dc75ed81e7274a4c8fc9d30ae3c50556 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/dc75ed81e7274a4c8fc9d30ae3c50556 2023-09-29T06:14:05,960 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/39aba1f938054a6e957bf82487842817 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/39aba1f938054a6e957bf82487842817 2023-09-29T06:14:05,961 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/a2a0e20e60674e0aa39d7d8dbf9f007d to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/a2a0e20e60674e0aa39d7d8dbf9f007d 2023-09-29T06:14:05,962 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/a1356accbe344dd198ead53dd1da6bed to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/a1356accbe344dd198ead53dd1da6bed 2023-09-29T06:14:05,964 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/7b23583faefa45929d0e787c3296c92e to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/7b23583faefa45929d0e787c3296c92e 2023-09-29T06:14:05,965 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/e9bd2b6eb08e4e62bc101f48c1636b8a to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/e9bd2b6eb08e4e62bc101f48c1636b8a 2023-09-29T06:14:05,967 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/8d0b29d3d0b143aaa452583c001ce668 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/8d0b29d3d0b143aaa452583c001ce668 2023-09-29T06:14:05,968 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/4b4ce7d6d60b46b99698f597cb331aad to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/4b4ce7d6d60b46b99698f597cb331aad 2023-09-29T06:14:05,969 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/2890bf40263b4b7e86e9bb811e28ff4b to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/2890bf40263b4b7e86e9bb811e28ff4b 2023-09-29T06:14:05,970 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/b695634ad3fb4642ab75deaf324386c8 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/b695634ad3fb4642ab75deaf324386c8 2023-09-29T06:14:05,971 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/6712827553864291b6437d5cabffb9d1 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/6712827553864291b6437d5cabffb9d1 2023-09-29T06:14:05,973 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/3e0e2153753440468db40372d268bdee to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/3e0e2153753440468db40372d268bdee 2023-09-29T06:14:05,974 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/d1c6ff4fc86d4fcfa94152d97db47d2a to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/d1c6ff4fc86d4fcfa94152d97db47d2a 2023-09-29T06:14:05,975 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/841bf539ffad45d28bad0935f07d5ef1 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/841bf539ffad45d28bad0935f07d5ef1 2023-09-29T06:14:05,977 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/fb4554ca49274424a35c1a468afc0c32 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/fb4554ca49274424a35c1a468afc0c32 2023-09-29T06:14:05,979 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/8e897e1875974cd59bab8c4abf91f6e5 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/8e897e1875974cd59bab8c4abf91f6e5 2023-09-29T06:14:05,980 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/2ca492b23399499abf9424f420d19b75 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/2ca492b23399499abf9424f420d19b75 2023-09-29T06:14:05,981 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/dce408cf71f74e999c731d8d8d54847c to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/dce408cf71f74e999c731d8d8d54847c 2023-09-29T06:14:05,982 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/34bd0b053b8a4f468f1fa1773f49b285 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/34bd0b053b8a4f468f1fa1773f49b285 2023-09-29T06:14:05,983 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/2da76a3ea0c44799bfa6bc70d6e509d0 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/2da76a3ea0c44799bfa6bc70d6e509d0 2023-09-29T06:14:05,984 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/0a7aca788ab34c5797fdc9a787b0edb7 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/0a7aca788ab34c5797fdc9a787b0edb7 2023-09-29T06:14:05,986 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/ea9180622cf042cda8999f5de5e0cf97 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/ea9180622cf042cda8999f5de5e0cf97 2023-09-29T06:14:05,987 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/dcc89245f12842d6a16981118411b59d to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/dcc89245f12842d6a16981118411b59d 2023-09-29T06:14:05,988 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/4c8b4b192df04f5caa338eff3a01287e to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/4c8b4b192df04f5caa338eff3a01287e 2023-09-29T06:14:05,989 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/9fa06d88873a48d4a1d94a3ff383380f to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/9fa06d88873a48d4a1d94a3ff383380f 2023-09-29T06:14:05,990 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/25bb783c113d43a88d7b878e3c127026 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/25bb783c113d43a88d7b878e3c127026 2023-09-29T06:14:05,991 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/66f8914a3047450ab9be270684153626 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/66f8914a3047450ab9be270684153626 2023-09-29T06:14:05,992 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/6c60e803bbcc4799bbf8f3a31595a3d8 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/6c60e803bbcc4799bbf8f3a31595a3d8 2023-09-29T06:14:05,995 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/e7e28f8579a842f78b33e3affd5af2c5, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/9380b95dfa3049dd96058607cbe0f233, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/2709eaf23a5844b7b41dc7d9067875b5, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/0c4f8455e7d14c1a8c2461f0f5158987, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/cc0b70518ae34bddbaf5b1917b381527, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/4e3967762ca04124bd413245751139da, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/3fad4c170ce24039a8f25a2d4e31cd37, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/1b3e450cfdbe4ca1917600b4e2cc6639, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/c193c67c3e61499ba003bfc55f347311, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/7bb1145b4be749f4a9acac33c1bf4d38, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/340cdfa54bd24ae6a863c7546b841b97, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/2fa8a2ab04d54a968000addaff7025e6, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/2c0cf557040b4fa5911c8bc1f640ec85, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/2071f61eb8ec4752a39f011ce534f4b0, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/85a33cebb05045c2858afc4be5c4ff81, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/d2361f6a1d6747dbbf32477310d1540c, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/45fee92478294bdaa44fcbb37c41a211, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/e5dcea97d3c04e51b3cffa5f68ba16e4, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/546d55e150e349ffa40a977db87d14ff, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/c84d8d7af6564a58bf99ad0be1940060, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/daf0bebb32d5458eb99f97b12f0cf986, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/12b698a8f39a4ae99c9a11379f5c8eee, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/064e9fb169f24dcaa50e021fc1207a94, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/93cf597f38d3482aa6e6a8df0f4546db, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/cbabfb10069f48d0a50c5452ea1689e6, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/f105a1e0990d4c538f61549130f9046d, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/ff8b2f2554d94bc782a5f73f267569b3, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/7d9d888ea0844f588275d0e1791499ae, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/080c2b9daeeb4a45829a2390c521a05d, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/336f3cd52546455ebc0cb6bd2db2b949, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/65f9f33e4bdc477681ca2d88db590655] to archive 2023-09-29T06:14:05,996 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2023-09-29T06:14:05,997 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/e7e28f8579a842f78b33e3affd5af2c5 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/e7e28f8579a842f78b33e3affd5af2c5 2023-09-29T06:14:05,998 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/9380b95dfa3049dd96058607cbe0f233 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/9380b95dfa3049dd96058607cbe0f233 2023-09-29T06:14:06,000 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/2709eaf23a5844b7b41dc7d9067875b5 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/2709eaf23a5844b7b41dc7d9067875b5 2023-09-29T06:14:06,002 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/0c4f8455e7d14c1a8c2461f0f5158987 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/0c4f8455e7d14c1a8c2461f0f5158987 2023-09-29T06:14:06,003 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/cc0b70518ae34bddbaf5b1917b381527 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/cc0b70518ae34bddbaf5b1917b381527 2023-09-29T06:14:06,004 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/4e3967762ca04124bd413245751139da to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/4e3967762ca04124bd413245751139da 2023-09-29T06:14:06,005 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/3fad4c170ce24039a8f25a2d4e31cd37 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/3fad4c170ce24039a8f25a2d4e31cd37 2023-09-29T06:14:06,007 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/1b3e450cfdbe4ca1917600b4e2cc6639 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/1b3e450cfdbe4ca1917600b4e2cc6639 2023-09-29T06:14:06,008 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/c193c67c3e61499ba003bfc55f347311 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/c193c67c3e61499ba003bfc55f347311 2023-09-29T06:14:06,009 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/7bb1145b4be749f4a9acac33c1bf4d38 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/7bb1145b4be749f4a9acac33c1bf4d38 2023-09-29T06:14:06,011 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/340cdfa54bd24ae6a863c7546b841b97 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/340cdfa54bd24ae6a863c7546b841b97 2023-09-29T06:14:06,012 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/2fa8a2ab04d54a968000addaff7025e6 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/2fa8a2ab04d54a968000addaff7025e6 2023-09-29T06:14:06,014 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/2c0cf557040b4fa5911c8bc1f640ec85 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/2c0cf557040b4fa5911c8bc1f640ec85 2023-09-29T06:14:06,015 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/2071f61eb8ec4752a39f011ce534f4b0 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/2071f61eb8ec4752a39f011ce534f4b0 2023-09-29T06:14:06,016 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/85a33cebb05045c2858afc4be5c4ff81 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/85a33cebb05045c2858afc4be5c4ff81 2023-09-29T06:14:06,018 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/d2361f6a1d6747dbbf32477310d1540c to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/d2361f6a1d6747dbbf32477310d1540c 2023-09-29T06:14:06,019 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/45fee92478294bdaa44fcbb37c41a211 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/45fee92478294bdaa44fcbb37c41a211 2023-09-29T06:14:06,021 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/e5dcea97d3c04e51b3cffa5f68ba16e4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/e5dcea97d3c04e51b3cffa5f68ba16e4 2023-09-29T06:14:06,022 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/546d55e150e349ffa40a977db87d14ff to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/546d55e150e349ffa40a977db87d14ff 2023-09-29T06:14:06,024 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/c84d8d7af6564a58bf99ad0be1940060 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/c84d8d7af6564a58bf99ad0be1940060 2023-09-29T06:14:06,025 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/daf0bebb32d5458eb99f97b12f0cf986 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/daf0bebb32d5458eb99f97b12f0cf986 2023-09-29T06:14:06,026 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/12b698a8f39a4ae99c9a11379f5c8eee to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/12b698a8f39a4ae99c9a11379f5c8eee 2023-09-29T06:14:06,028 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/064e9fb169f24dcaa50e021fc1207a94 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/064e9fb169f24dcaa50e021fc1207a94 2023-09-29T06:14:06,029 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/93cf597f38d3482aa6e6a8df0f4546db to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/93cf597f38d3482aa6e6a8df0f4546db 2023-09-29T06:14:06,030 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/cbabfb10069f48d0a50c5452ea1689e6 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/cbabfb10069f48d0a50c5452ea1689e6 2023-09-29T06:14:06,031 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/f105a1e0990d4c538f61549130f9046d to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/f105a1e0990d4c538f61549130f9046d 2023-09-29T06:14:06,032 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/ff8b2f2554d94bc782a5f73f267569b3 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/ff8b2f2554d94bc782a5f73f267569b3 2023-09-29T06:14:06,033 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/7d9d888ea0844f588275d0e1791499ae to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/7d9d888ea0844f588275d0e1791499ae 2023-09-29T06:14:06,034 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/080c2b9daeeb4a45829a2390c521a05d to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/080c2b9daeeb4a45829a2390c521a05d 2023-09-29T06:14:06,035 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/336f3cd52546455ebc0cb6bd2db2b949 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/336f3cd52546455ebc0cb6bd2db2b949 2023-09-29T06:14:06,037 DEBUG [StoreCloser-TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/65f9f33e4bdc477681ca2d88db590655 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/65f9f33e4bdc477681ca2d88db590655 2023-09-29T06:14:06,041 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/recovered.edits/486.seqid, newMaxSeqId=486, maxSeqId=4 2023-09-29T06:14:06,043 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegion(1909): Closed TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4. 2023-09-29T06:14:06,043 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegion(1622): Region close journal for 1668fccb65cfe5ce16ea12b0cc0df5e4: 2023-09-29T06:14:06,044 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] handler.UnassignRegionHandler(163): Closed 1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:14:06,044 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=67 updating hbase:meta row=1668fccb65cfe5ce16ea12b0cc0df5e4, regionState=CLOSED 2023-09-29T06:14:06,047 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=68, resume processing ppid=67 2023-09-29T06:14:06,047 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=68, ppid=67, state=SUCCESS; CloseRegionProcedure 1668fccb65cfe5ce16ea12b0cc0df5e4, server=jenkins-hbase3.apache.org,39783,1695967978138 in 813 msec 2023-09-29T06:14:06,048 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=67, resume processing ppid=66 2023-09-29T06:14:06,048 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=67, ppid=66, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=1668fccb65cfe5ce16ea12b0cc0df5e4, UNASSIGN in 818 msec 2023-09-29T06:14:06,049 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968046049"}]},"ts":"1695968046049"} 2023-09-29T06:14:06,050 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2023-09-29T06:14:06,052 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(305): Set TestAcidGuarantees to state=DISABLED 2023-09-29T06:14:06,054 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=66, state=SUCCESS; DisableTableProcedure table=TestAcidGuarantees in 834 msec 2023-09-29T06:14:06,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=66 2023-09-29T06:14:06,328 INFO [Listener at localhost/42843 {}] client.HBaseAdmin$TableFuture(3677): Operation: DISABLE, Table Name: default:TestAcidGuarantees, procId: 66 completed 2023-09-29T06:14:06,329 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$5(2448): Client=jenkins//172.31.12.81 delete TestAcidGuarantees 2023-09-29T06:14:06,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=69, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=TestAcidGuarantees 2023-09-29T06:14:06,332 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=69, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2023-09-29T06:14:06,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=69 2023-09-29T06:14:06,332 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=69, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2023-09-29T06:14:06,334 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:14:06,338 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A, FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B, FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C, FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/recovered.edits] 2023-09-29T06:14:06,342 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/238a584958b5430fabea55e251f39c31 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/238a584958b5430fabea55e251f39c31 2023-09-29T06:14:06,344 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/6d7664195c4c43a2b04a1d71fc644836 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/6d7664195c4c43a2b04a1d71fc644836 2023-09-29T06:14:06,346 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/b1c1986e04dc431cb284332c5b74e868 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/A/b1c1986e04dc431cb284332c5b74e868 2023-09-29T06:14:06,348 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/31d7fa1fd7234ed1955fbf96a012a7dc to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/31d7fa1fd7234ed1955fbf96a012a7dc 2023-09-29T06:14:06,351 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/e44785e0ad4d401fa77dbe77f7c4ad59 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/e44785e0ad4d401fa77dbe77f7c4ad59 2023-09-29T06:14:06,352 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/f8435501d558436da342d1c174894b8c to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/B/f8435501d558436da342d1c174894b8c 2023-09-29T06:14:06,355 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/10e2b799a60842b6a52ca5ccba0c6a7f to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/10e2b799a60842b6a52ca5ccba0c6a7f 2023-09-29T06:14:06,357 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/9b807447882f49faa0c96d8f9184be4c to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/9b807447882f49faa0c96d8f9184be4c 2023-09-29T06:14:06,358 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/d6dc7f54197649f9b51f61b2fe1f4a53 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/C/d6dc7f54197649f9b51f61b2fe1f4a53 2023-09-29T06:14:06,361 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/recovered.edits/486.seqid to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4/recovered.edits/486.seqid 2023-09-29T06:14:06,362 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:14:06,362 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(293): Archived TestAcidGuarantees regions 2023-09-29T06:14:06,363 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2023-09-29T06:14:06,363 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2023-09-29T06:14:06,370 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929024c14c243664a30b22c86ffcbc8eb1e_1668fccb65cfe5ce16ea12b0cc0df5e4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929024c14c243664a30b22c86ffcbc8eb1e_1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:14:06,371 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023092913b422afda3f40bb9c27bacbf1ed15dd_1668fccb65cfe5ce16ea12b0cc0df5e4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023092913b422afda3f40bb9c27bacbf1ed15dd_1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:14:06,373 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309291cfaf382debc41d5b527ee87036fe8bb_1668fccb65cfe5ce16ea12b0cc0df5e4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309291cfaf382debc41d5b527ee87036fe8bb_1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:14:06,374 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023092920016fa647d34fcf8ade822f8c7de576_1668fccb65cfe5ce16ea12b0cc0df5e4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023092920016fa647d34fcf8ade822f8c7de576_1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:14:06,376 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929205ef05d47f64cf6ab0467c191341b4f_1668fccb65cfe5ce16ea12b0cc0df5e4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929205ef05d47f64cf6ab0467c191341b4f_1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:14:06,377 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929349628894023402f87b3bcb0f1104215_1668fccb65cfe5ce16ea12b0cc0df5e4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929349628894023402f87b3bcb0f1104215_1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:14:06,379 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309294bdd738d8853443b86eb1dbcb5a1886a_1668fccb65cfe5ce16ea12b0cc0df5e4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309294bdd738d8853443b86eb1dbcb5a1886a_1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:14:06,381 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309294c3e8997257b4371a82a512877215d14_1668fccb65cfe5ce16ea12b0cc0df5e4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309294c3e8997257b4371a82a512877215d14_1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:14:06,382 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309294ebc41e3800841dba4cd51492e9fa0fe_1668fccb65cfe5ce16ea12b0cc0df5e4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309294ebc41e3800841dba4cd51492e9fa0fe_1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:14:06,384 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309295cf172ba605c4b39b6c2c5fa9abb417e_1668fccb65cfe5ce16ea12b0cc0df5e4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309295cf172ba605c4b39b6c2c5fa9abb417e_1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:14:06,385 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309296041c21273f24023bf13f8016c83e89c_1668fccb65cfe5ce16ea12b0cc0df5e4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309296041c21273f24023bf13f8016c83e89c_1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:14:06,386 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309296facf53b66c54cbab05b2bf75daf4abb_1668fccb65cfe5ce16ea12b0cc0df5e4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309296facf53b66c54cbab05b2bf75daf4abb_1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:14:06,387 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023092975b5ccec6311487f8325f08720d89842_1668fccb65cfe5ce16ea12b0cc0df5e4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023092975b5ccec6311487f8325f08720d89842_1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:14:06,389 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309298e05a4e1388e4343b0b4090032bda0cc_1668fccb65cfe5ce16ea12b0cc0df5e4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309298e05a4e1388e4343b0b4090032bda0cc_1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:14:06,391 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309299049f4950f624bc49cf365964506ef49_1668fccb65cfe5ce16ea12b0cc0df5e4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309299049f4950f624bc49cf365964506ef49_1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:14:06,392 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023092994285a1b0ddc4cfaa8ecb5ac986ee07f_1668fccb65cfe5ce16ea12b0cc0df5e4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023092994285a1b0ddc4cfaa8ecb5ac986ee07f_1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:14:06,393 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309299b3d156444d44c9a91d8ecd9d6a5128d_1668fccb65cfe5ce16ea12b0cc0df5e4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309299b3d156444d44c9a91d8ecd9d6a5128d_1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:14:06,395 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929b0b6620a0178413bbfe437e3a2114976_1668fccb65cfe5ce16ea12b0cc0df5e4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929b0b6620a0178413bbfe437e3a2114976_1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:14:06,397 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929b7c44e0e0fce41cd89decd5d5cde05b7_1668fccb65cfe5ce16ea12b0cc0df5e4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929b7c44e0e0fce41cd89decd5d5cde05b7_1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:14:06,399 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929ba4817f815274af6801b5fcf54311055_1668fccb65cfe5ce16ea12b0cc0df5e4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929ba4817f815274af6801b5fcf54311055_1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:14:06,401 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929cec404f530d842179f58439b0ee1bdda_1668fccb65cfe5ce16ea12b0cc0df5e4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929cec404f530d842179f58439b0ee1bdda_1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:14:06,402 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929d4d575eb6a20451fa37ac172e1e26ad6_1668fccb65cfe5ce16ea12b0cc0df5e4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929d4d575eb6a20451fa37ac172e1e26ad6_1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:14:06,404 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929f305958024664b1fa9daecafc3ef49eb_1668fccb65cfe5ce16ea12b0cc0df5e4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929f305958024664b1fa9daecafc3ef49eb_1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:14:06,406 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929f4c12b9649ae45299330baa06355c0c1_1668fccb65cfe5ce16ea12b0cc0df5e4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929f4c12b9649ae45299330baa06355c0c1_1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:14:06,411 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929f4c9931d56f242a9ac21379afc9b8cec_1668fccb65cfe5ce16ea12b0cc0df5e4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929f4c9931d56f242a9ac21379afc9b8cec_1668fccb65cfe5ce16ea12b0cc0df5e4 2023-09-29T06:14:06,412 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2023-09-29T06:14:06,415 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=69, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2023-09-29T06:14:06,417 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2023-09-29T06:14:06,424 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(387): Removing 'TestAcidGuarantees' descriptor. 2023-09-29T06:14:06,425 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=69, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2023-09-29T06:14:06,425 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(377): Removing 'TestAcidGuarantees' from region states. 2023-09-29T06:14:06,425 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1695968046425"}]},"ts":"9223372036854775807"} 2023-09-29T06:14:06,427 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-09-29T06:14:06,427 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 1668fccb65cfe5ce16ea12b0cc0df5e4, NAME => 'TestAcidGuarantees,,1695968012792.1668fccb65cfe5ce16ea12b0cc0df5e4.', STARTKEY => '', ENDKEY => ''}] 2023-09-29T06:14:06,427 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(381): Marking 'TestAcidGuarantees' as deleted. 2023-09-29T06:14:06,428 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1695968046427"}]},"ts":"9223372036854775807"} 2023-09-29T06:14:06,429 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1678): Deleted table TestAcidGuarantees state from META 2023-09-29T06:14:06,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=69 2023-09-29T06:14:06,433 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(127): Finished pid=69, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2023-09-29T06:14:06,434 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=69, state=SUCCESS; DeleteTableProcedure table=TestAcidGuarantees in 104 msec 2023-09-29T06:14:06,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=69 2023-09-29T06:14:06,634 INFO [Listener at localhost/42843 {}] client.HBaseAdmin$TableFuture(3677): Operation: DELETE, Table Name: default:TestAcidGuarantees, procId: 69 completed 2023-09-29T06:14:06,649 INFO [Listener at localhost/42843 {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithAdaptivePolicy#testMobMixedAtomicity Thread=218 (was 220), OpenFileDescriptor=562 (was 570), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=712 (was 659) - SystemLoadAverage LEAK? -, ProcessCount=171 (was 171), AvailableMemoryMB=1815 (was 2035) 2023-09-29T06:14:06,664 INFO [Listener at localhost/42843 {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithAdaptivePolicy#testGetAtomicity Thread=218, OpenFileDescriptor=562, MaxFileDescriptor=60000, SystemLoadAverage=712, ProcessCount=171, AvailableMemoryMB=1814 2023-09-29T06:14:06,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] util.TableDescriptorChecker(342): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2023-09-29T06:14:06,671 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'ADAPTIVE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-29T06:14:06,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=70, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestAcidGuarantees 2023-09-29T06:14:06,674 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=70, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2023-09-29T06:14:06,674 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:14:06,674 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 70 2023-09-29T06:14:06,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=70 2023-09-29T06:14:06,675 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=70, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-29T06:14:06,683 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'ADAPTIVE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a 2023-09-29T06:14:06,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=70 2023-09-29T06:14:06,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=70 2023-09-29T06:14:07,089 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(888): Instantiated TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:14:07,089 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1668): Closing eefd39c60cf58e105fccf6c634940207, disabling compactions & flushes 2023-09-29T06:14:07,089 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1690): Closing region TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:07,089 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:07,089 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. after waiting 0 ms 2023-09-29T06:14:07,089 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:07,089 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1909): Closed TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:07,089 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1622): Region close journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:07,091 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=70, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2023-09-29T06:14:07,091 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1695968047091"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1695968047091"}]},"ts":"1695968047091"} 2023-09-29T06:14:07,092 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-29T06:14:07,092 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=70, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-29T06:14:07,093 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968047092"}]},"ts":"1695968047092"} 2023-09-29T06:14:07,093 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2023-09-29T06:14:07,099 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=eefd39c60cf58e105fccf6c634940207, ASSIGN}] 2023-09-29T06:14:07,100 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=71, ppid=70, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=eefd39c60cf58e105fccf6c634940207, ASSIGN 2023-09-29T06:14:07,102 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=71, ppid=70, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=eefd39c60cf58e105fccf6c634940207, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,39783,1695967978138; forceNewPlan=false, retain=false 2023-09-29T06:14:07,254 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=71 updating hbase:meta row=eefd39c60cf58e105fccf6c634940207, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:07,256 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=72, ppid=71, state=RUNNABLE; OpenRegionProcedure eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138}] 2023-09-29T06:14:07,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=70 2023-09-29T06:14:07,419 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] handler.AssignRegionHandler(130): Open TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:07,419 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7254): Opening region: {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} 2023-09-29T06:14:07,420 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees eefd39c60cf58e105fccf6c634940207 2023-09-29T06:14:07,420 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(888): Instantiated TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:14:07,420 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7296): checking encryption for eefd39c60cf58e105fccf6c634940207 2023-09-29T06:14:07,420 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7299): checking classloading for eefd39c60cf58e105fccf6c634940207 2023-09-29T06:14:07,421 INFO [StoreOpener-eefd39c60cf58e105fccf6c634940207-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region eefd39c60cf58e105fccf6c634940207 2023-09-29T06:14:07,423 INFO [StoreOpener-eefd39c60cf58e105fccf6c634940207-1 {}] regionserver.CompactingMemStore(121): Store=A, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2023-09-29T06:14:07,423 INFO [StoreOpener-eefd39c60cf58e105fccf6c634940207-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eefd39c60cf58e105fccf6c634940207 columnFamilyName A 2023-09-29T06:14:07,423 DEBUG [StoreOpener-eefd39c60cf58e105fccf6c634940207-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:14:07,423 INFO [StoreOpener-eefd39c60cf58e105fccf6c634940207-1 {}] regionserver.HStore(324): Store=eefd39c60cf58e105fccf6c634940207/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:14:07,424 INFO [StoreOpener-eefd39c60cf58e105fccf6c634940207-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region eefd39c60cf58e105fccf6c634940207 2023-09-29T06:14:07,425 INFO [StoreOpener-eefd39c60cf58e105fccf6c634940207-1 {}] regionserver.CompactingMemStore(121): Store=B, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2023-09-29T06:14:07,425 INFO [StoreOpener-eefd39c60cf58e105fccf6c634940207-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eefd39c60cf58e105fccf6c634940207 columnFamilyName B 2023-09-29T06:14:07,425 DEBUG [StoreOpener-eefd39c60cf58e105fccf6c634940207-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:14:07,425 INFO [StoreOpener-eefd39c60cf58e105fccf6c634940207-1 {}] regionserver.HStore(324): Store=eefd39c60cf58e105fccf6c634940207/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:14:07,425 INFO [StoreOpener-eefd39c60cf58e105fccf6c634940207-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region eefd39c60cf58e105fccf6c634940207 2023-09-29T06:14:07,427 INFO [StoreOpener-eefd39c60cf58e105fccf6c634940207-1 {}] regionserver.CompactingMemStore(121): Store=C, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2023-09-29T06:14:07,427 INFO [StoreOpener-eefd39c60cf58e105fccf6c634940207-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eefd39c60cf58e105fccf6c634940207 columnFamilyName C 2023-09-29T06:14:07,427 DEBUG [StoreOpener-eefd39c60cf58e105fccf6c634940207-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:14:07,428 INFO [StoreOpener-eefd39c60cf58e105fccf6c634940207-1 {}] regionserver.HStore(324): Store=eefd39c60cf58e105fccf6c634940207/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:14:07,428 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1169): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:07,429 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207 2023-09-29T06:14:07,429 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207 2023-09-29T06:14:07,431 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2023-09-29T06:14:07,432 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1076): writing seq id for eefd39c60cf58e105fccf6c634940207 2023-09-29T06:14:07,446 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-29T06:14:07,447 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1093): Opened eefd39c60cf58e105fccf6c634940207; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69875450, jitterRate=0.04122534394264221}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2023-09-29T06:14:07,448 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(992): Region open journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:07,448 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2556): Post open deploy tasks for TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207., pid=72, masterSystemTime=1695968047408 2023-09-29T06:14:07,450 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2583): Finished post open deploy task for TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:07,450 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] handler.AssignRegionHandler(158): Opened TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:07,450 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=71 updating hbase:meta row=eefd39c60cf58e105fccf6c634940207, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:07,452 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=72, resume processing ppid=71 2023-09-29T06:14:07,452 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=72, ppid=71, state=SUCCESS; OpenRegionProcedure eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 in 196 msec 2023-09-29T06:14:07,454 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=71, resume processing ppid=70 2023-09-29T06:14:07,454 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=71, ppid=70, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=eefd39c60cf58e105fccf6c634940207, ASSIGN in 353 msec 2023-09-29T06:14:07,455 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=70, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-29T06:14:07,455 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968047455"}]},"ts":"1695968047455"} 2023-09-29T06:14:07,456 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2023-09-29T06:14:07,458 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=70, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2023-09-29T06:14:07,459 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=70, state=SUCCESS; CreateTableProcedure table=TestAcidGuarantees in 787 msec 2023-09-29T06:14:07,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=70 2023-09-29T06:14:07,779 INFO [Listener at localhost/42843 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: default:TestAcidGuarantees, procId: 70 completed 2023-09-29T06:14:07,781 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x01357904 to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:14:07,790 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@301ba600, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:14:07,791 DEBUG [Listener at localhost/42843 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:14:07,793 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:40970, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:14:07,795 DEBUG [Listener at localhost/42843 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-29T06:14:07,796 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:34198, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-29T06:14:07,798 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x5017cdbc to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:14:07,805 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a4073e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:14:07,807 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x71e339c4 to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:14:07,811 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ff77069, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:14:07,812 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x46ffdc41 to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:14:07,817 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e3af043, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:14:07,818 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x62eb7ea8 to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:14:07,825 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5acebb60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:14:07,825 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x7c744127 to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:14:07,834 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45537e3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:14:07,835 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x681d2c78 to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:14:07,839 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36e105c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:14:07,840 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x55e0ed62 to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:14:07,844 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@541f754a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:14:07,846 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0ce1917b to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:14:07,853 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e38e678, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:14:07,854 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x6e7b70bf to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:14:07,860 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e114e42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:14:07,861 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x15adc103 to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:14:07,873 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2066b581, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:14:07,883 DEBUG [hconnection-0x6c33e03a-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:14:07,884 DEBUG [hconnection-0x50fef9ac-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:14:07,885 DEBUG [hconnection-0x3e21414c-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:14:07,885 DEBUG [hconnection-0xc0c8378-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:14:07,885 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:14:07,885 DEBUG [hconnection-0x15c4341a-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:14:07,887 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:40992, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:14:07,887 DEBUG [hconnection-0x588e4c02-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:14:07,888 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:41018, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:14:07,888 DEBUG [hconnection-0x62944f86-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:14:07,889 DEBUG [hconnection-0x2c5f9f9c-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:14:07,890 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:41050, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:14:07,896 DEBUG [hconnection-0xf632777-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:14:07,898 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:41006, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:14:07,898 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:41020, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:14:07,898 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:41054, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:14:07,899 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:40984, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:14:07,899 DEBUG [hconnection-0x78d502d2-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:14:07,899 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:41008, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:14:07,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=73, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=73, table=TestAcidGuarantees 2023-09-29T06:14:07,899 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:41034, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:14:07,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=73 2023-09-29T06:14:07,905 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=73, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=73, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:14:07,905 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:41062, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:14:07,906 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=73, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=73, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:14:07,906 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=74, ppid=73, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:14:07,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on eefd39c60cf58e105fccf6c634940207 2023-09-29T06:14:07,924 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing eefd39c60cf58e105fccf6c634940207 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2023-09-29T06:14:07,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=A 2023-09-29T06:14:07,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:07,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=B 2023-09-29T06:14:07,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:07,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=C 2023-09-29T06:14:07,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:07,949 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/e724149fa96c47eea4eee887cad53c7e is 50, key is test_row_0/A:col10/1695968047920/Put/seqid=0 2023-09-29T06:14:07,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:07,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968107960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:07,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:07,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968107961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:07,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:07,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968107965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:07,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:07,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968107967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:07,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:07,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41054 deadline: 1695968107967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:07,973 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/e724149fa96c47eea4eee887cad53c7e 2023-09-29T06:14:08,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=73 2023-09-29T06:14:08,061 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/813aa866615746fd93a60db92e8044df is 50, key is test_row_0/B:col10/1695968047920/Put/seqid=0 2023-09-29T06:14:08,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:08,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968108069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:08,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:08,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968108069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:08,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:08,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968108070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:08,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=74 2023-09-29T06:14:08,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:08,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. as already flushing 2023-09-29T06:14:08,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:08,072 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] handler.RSProcedureHandler(53): pid=74 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:08,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=74 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:08,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:08,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=74 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:08,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968108072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:08,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:08,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41054 deadline: 1695968108073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:08,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=73 2023-09-29T06:14:08,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=74 2023-09-29T06:14:08,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:08,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. as already flushing 2023-09-29T06:14:08,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:08,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=74}] handler.RSProcedureHandler(53): pid=74 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:08,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=74 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:08,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=74 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:08,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:08,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968108274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:08,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:08,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968108275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:08,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:08,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968108275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:08,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:08,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41054 deadline: 1695968108276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:08,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:08,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968108279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:08,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=74 2023-09-29T06:14:08,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:08,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. as already flushing 2023-09-29T06:14:08,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:08,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=74}] handler.RSProcedureHandler(53): pid=74 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:08,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=74 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:08,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=74 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:08,506 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/813aa866615746fd93a60db92e8044df 2023-09-29T06:14:08,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=73 2023-09-29T06:14:08,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=74 2023-09-29T06:14:08,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:08,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. as already flushing 2023-09-29T06:14:08,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:08,535 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] handler.RSProcedureHandler(53): pid=74 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:08,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=74 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:08,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/a559532c42054b8fbd9af3c6e16eb3cf is 50, key is test_row_0/C:col10/1695968047920/Put/seqid=0 2023-09-29T06:14:08,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=74 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:08,552 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/a559532c42054b8fbd9af3c6e16eb3cf 2023-09-29T06:14:08,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/e724149fa96c47eea4eee887cad53c7e as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/e724149fa96c47eea4eee887cad53c7e 2023-09-29T06:14:08,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/e724149fa96c47eea4eee887cad53c7e, entries=150, sequenceid=14, filesize=11.7 K 2023-09-29T06:14:08,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/813aa866615746fd93a60db92e8044df as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/813aa866615746fd93a60db92e8044df 2023-09-29T06:14:08,570 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/813aa866615746fd93a60db92e8044df, entries=150, sequenceid=14, filesize=11.7 K 2023-09-29T06:14:08,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/a559532c42054b8fbd9af3c6e16eb3cf as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/a559532c42054b8fbd9af3c6e16eb3cf 2023-09-29T06:14:08,576 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/a559532c42054b8fbd9af3c6e16eb3cf, entries=150, sequenceid=14, filesize=11.7 K 2023-09-29T06:14:08,577 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for eefd39c60cf58e105fccf6c634940207 in 653ms, sequenceid=14, compaction requested=false 2023-09-29T06:14:08,577 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2023-09-29T06:14:08,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:08,585 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing eefd39c60cf58e105fccf6c634940207 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2023-09-29T06:14:08,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on eefd39c60cf58e105fccf6c634940207 2023-09-29T06:14:08,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=A 2023-09-29T06:14:08,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:08,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=B 2023-09-29T06:14:08,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:08,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=C 2023-09-29T06:14:08,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:08,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:08,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:08,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968108594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:08,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968108594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:08,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:08,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968108594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:08,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:08,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968108596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:08,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:08,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41054 deadline: 1695968108597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:08,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/e0c8e0dbeb9544d2bf31a0bd98eaeaa5 is 50, key is test_row_0/A:col10/1695968047963/Put/seqid=0 2023-09-29T06:14:08,616 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/e0c8e0dbeb9544d2bf31a0bd98eaeaa5 2023-09-29T06:14:08,623 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/9f892ef609564eb1b45fdbf811a46fae is 50, key is test_row_0/B:col10/1695968047963/Put/seqid=0 2023-09-29T06:14:08,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=74 2023-09-29T06:14:08,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:08,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. as already flushing 2023-09-29T06:14:08,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:08,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=74}] handler.RSProcedureHandler(53): pid=74 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:08,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=74 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:08,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=74 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:08,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:08,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968108698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:08,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:08,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968108698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:08,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:08,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968108699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:08,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:08,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968108700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:08,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:08,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41054 deadline: 1695968108708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:08,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=74 2023-09-29T06:14:08,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:08,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. as already flushing 2023-09-29T06:14:08,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:08,842 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=74}] handler.RSProcedureHandler(53): pid=74 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:08,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=74 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:08,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=74 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:08,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:08,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968108901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:08,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:08,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968108902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:08,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:08,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968108903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:08,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:08,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968108903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:08,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:08,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41054 deadline: 1695968108911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:08,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=74 2023-09-29T06:14:08,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:08,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. as already flushing 2023-09-29T06:14:08,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:08,995 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] handler.RSProcedureHandler(53): pid=74 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:08,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=74 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:08,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=74 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:09,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=73 2023-09-29T06:14:09,037 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/9f892ef609564eb1b45fdbf811a46fae 2023-09-29T06:14:09,049 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/ddedb0a61b1f471ab68d8690326d532e is 50, key is test_row_0/C:col10/1695968047963/Put/seqid=0 2023-09-29T06:14:09,063 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/ddedb0a61b1f471ab68d8690326d532e 2023-09-29T06:14:09,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/e0c8e0dbeb9544d2bf31a0bd98eaeaa5 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/e0c8e0dbeb9544d2bf31a0bd98eaeaa5 2023-09-29T06:14:09,075 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/e0c8e0dbeb9544d2bf31a0bd98eaeaa5, entries=200, sequenceid=42, filesize=14.0 K 2023-09-29T06:14:09,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/9f892ef609564eb1b45fdbf811a46fae as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/9f892ef609564eb1b45fdbf811a46fae 2023-09-29T06:14:09,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/9f892ef609564eb1b45fdbf811a46fae, entries=150, sequenceid=42, filesize=11.7 K 2023-09-29T06:14:09,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/ddedb0a61b1f471ab68d8690326d532e as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/ddedb0a61b1f471ab68d8690326d532e 2023-09-29T06:14:09,093 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/ddedb0a61b1f471ab68d8690326d532e, entries=150, sequenceid=42, filesize=11.7 K 2023-09-29T06:14:09,105 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=40.25 KB/41220 for eefd39c60cf58e105fccf6c634940207 in 520ms, sequenceid=42, compaction requested=false 2023-09-29T06:14:09,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:09,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=74 2023-09-29T06:14:09,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:09,148 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.HRegion(2819): Flushing eefd39c60cf58e105fccf6c634940207 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2023-09-29T06:14:09,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=A 2023-09-29T06:14:09,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:09,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=B 2023-09-29T06:14:09,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:09,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=C 2023-09-29T06:14:09,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:09,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=74}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/b556a8fd5c9245cba9973b546ffb669b is 50, key is test_row_0/A:col10/1695968048595/Put/seqid=0 2023-09-29T06:14:09,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on eefd39c60cf58e105fccf6c634940207 2023-09-29T06:14:09,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. as already flushing 2023-09-29T06:14:09,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:09,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:09,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968109229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:09,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968109230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:09,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:09,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41054 deadline: 1695968109233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:09,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:09,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968109234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:09,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:09,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968109234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:09,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:09,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968109335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:09,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:09,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41054 deadline: 1695968109336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:09,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:09,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968109337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:09,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:09,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968109337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:09,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:09,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968109338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:09,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:09,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968109539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:09,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:09,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41054 deadline: 1695968109539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:09,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:09,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968109540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:09,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:09,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968109541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:09,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:09,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968109542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:09,565 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/b556a8fd5c9245cba9973b546ffb669b 2023-09-29T06:14:09,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=74}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/cc1fd449d57f4409b0ddfae3d32b3264 is 50, key is test_row_0/B:col10/1695968048595/Put/seqid=0 2023-09-29T06:14:09,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:09,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968109841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:09,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:09,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968109842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:09,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:09,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41054 deadline: 1695968109843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:09,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:09,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968109843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:09,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:09,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968109844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:09,985 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/cc1fd449d57f4409b0ddfae3d32b3264 2023-09-29T06:14:10,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=74}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/ab63331258dd4ae0ab2f9e03f971b50f is 50, key is test_row_0/C:col10/1695968048595/Put/seqid=0 2023-09-29T06:14:10,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=73 2023-09-29T06:14:10,022 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/ab63331258dd4ae0ab2f9e03f971b50f 2023-09-29T06:14:10,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/b556a8fd5c9245cba9973b546ffb669b as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/b556a8fd5c9245cba9973b546ffb669b 2023-09-29T06:14:10,033 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/b556a8fd5c9245cba9973b546ffb669b, entries=100, sequenceid=51, filesize=9.4 K 2023-09-29T06:14:10,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/cc1fd449d57f4409b0ddfae3d32b3264 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/cc1fd449d57f4409b0ddfae3d32b3264 2023-09-29T06:14:10,049 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/cc1fd449d57f4409b0ddfae3d32b3264, entries=100, sequenceid=51, filesize=9.4 K 2023-09-29T06:14:10,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/ab63331258dd4ae0ab2f9e03f971b50f as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/ab63331258dd4ae0ab2f9e03f971b50f 2023-09-29T06:14:10,058 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/ab63331258dd4ae0ab2f9e03f971b50f, entries=100, sequenceid=51, filesize=9.4 K 2023-09-29T06:14:10,059 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.HRegion(3022): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=167.72 KB/171750 for eefd39c60cf58e105fccf6c634940207 in 911ms, sequenceid=51, compaction requested=true 2023-09-29T06:14:10,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.HRegion(2520): Flush status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:10,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:10,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=74 2023-09-29T06:14:10,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=74 2023-09-29T06:14:10,072 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=74, resume processing ppid=73 2023-09-29T06:14:10,072 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=74, ppid=73, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1550 sec 2023-09-29T06:14:10,074 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=73, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=73, table=TestAcidGuarantees in 2.1870 sec 2023-09-29T06:14:10,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on eefd39c60cf58e105fccf6c634940207 2023-09-29T06:14:10,348 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing eefd39c60cf58e105fccf6c634940207 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2023-09-29T06:14:10,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=A 2023-09-29T06:14:10,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:10,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=B 2023-09-29T06:14:10,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:10,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=C 2023-09-29T06:14:10,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:10,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:10,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41054 deadline: 1695968110350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:10,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:10,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968110352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:10,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:10,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968110353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:10,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:10,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968110353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:10,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:10,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968110358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:10,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/6633ac64727244dfbc57784591c40e9e is 50, key is test_row_0/A:col10/1695968050348/Put/seqid=0 2023-09-29T06:14:10,375 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/6633ac64727244dfbc57784591c40e9e 2023-09-29T06:14:10,397 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/44ea540e8b574931ae12c754acac4881 is 50, key is test_row_0/B:col10/1695968050348/Put/seqid=0 2023-09-29T06:14:10,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:10,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968110457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:10,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:10,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:10,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968110457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:10,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968110457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:10,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:10,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968110460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:10,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:10,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968110661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:10,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:10,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968110661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:10,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:10,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968110662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:10,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:10,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968110664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:10,801 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/44ea540e8b574931ae12c754acac4881 2023-09-29T06:14:10,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/929d6588860645fc81eb8427973bcde5 is 50, key is test_row_0/C:col10/1695968050348/Put/seqid=0 2023-09-29T06:14:10,833 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/929d6588860645fc81eb8427973bcde5 2023-09-29T06:14:10,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/6633ac64727244dfbc57784591c40e9e as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/6633ac64727244dfbc57784591c40e9e 2023-09-29T06:14:10,842 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/6633ac64727244dfbc57784591c40e9e, entries=150, sequenceid=81, filesize=11.7 K 2023-09-29T06:14:10,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/44ea540e8b574931ae12c754acac4881 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/44ea540e8b574931ae12c754acac4881 2023-09-29T06:14:10,860 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/44ea540e8b574931ae12c754acac4881, entries=150, sequenceid=81, filesize=11.7 K 2023-09-29T06:14:10,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/929d6588860645fc81eb8427973bcde5 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/929d6588860645fc81eb8427973bcde5 2023-09-29T06:14:10,871 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/929d6588860645fc81eb8427973bcde5, entries=150, sequenceid=81, filesize=11.7 K 2023-09-29T06:14:10,872 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=26.84 KB/27480 for eefd39c60cf58e105fccf6c634940207 in 523ms, sequenceid=81, compaction requested=true 2023-09-29T06:14:10,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:10,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eefd39c60cf58e105fccf6c634940207:A, priority=-2147483648, current under compaction store size is 1 2023-09-29T06:14:10,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:10,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eefd39c60cf58e105fccf6c634940207:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:14:10,872 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-29T06:14:10,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:10,872 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-29T06:14:10,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eefd39c60cf58e105fccf6c634940207:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:14:10,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:14:10,873 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47932 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-29T06:14:10,873 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): eefd39c60cf58e105fccf6c634940207/A is initiating minor compaction (all files) 2023-09-29T06:14:10,873 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45592 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-29T06:14:10,873 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of eefd39c60cf58e105fccf6c634940207/A in TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:10,873 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): eefd39c60cf58e105fccf6c634940207/B is initiating minor compaction (all files) 2023-09-29T06:14:10,873 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/e724149fa96c47eea4eee887cad53c7e, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/e0c8e0dbeb9544d2bf31a0bd98eaeaa5, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/b556a8fd5c9245cba9973b546ffb669b, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/6633ac64727244dfbc57784591c40e9e] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp, totalSize=46.8 K 2023-09-29T06:14:10,873 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of eefd39c60cf58e105fccf6c634940207/B in TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:10,873 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/813aa866615746fd93a60db92e8044df, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/9f892ef609564eb1b45fdbf811a46fae, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/cc1fd449d57f4409b0ddfae3d32b3264, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/44ea540e8b574931ae12c754acac4881] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp, totalSize=44.5 K 2023-09-29T06:14:10,874 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting e724149fa96c47eea4eee887cad53c7e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1695968047920 2023-09-29T06:14:10,874 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 813aa866615746fd93a60db92e8044df, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1695968047920 2023-09-29T06:14:10,874 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting e0c8e0dbeb9544d2bf31a0bd98eaeaa5, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1695968047963 2023-09-29T06:14:10,874 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 9f892ef609564eb1b45fdbf811a46fae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1695968047963 2023-09-29T06:14:10,874 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting b556a8fd5c9245cba9973b546ffb669b, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1695968048595 2023-09-29T06:14:10,875 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting cc1fd449d57f4409b0ddfae3d32b3264, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1695968048595 2023-09-29T06:14:10,875 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 6633ac64727244dfbc57784591c40e9e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1695968049233 2023-09-29T06:14:10,875 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 44ea540e8b574931ae12c754acac4881, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1695968049233 2023-09-29T06:14:10,900 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eefd39c60cf58e105fccf6c634940207#A#compaction#245 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:14:10,900 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eefd39c60cf58e105fccf6c634940207#B#compaction#244 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:14:10,900 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/ab44d8773a994b7d89a08e0ff2b0b864 is 50, key is test_row_0/A:col10/1695968050348/Put/seqid=0 2023-09-29T06:14:10,900 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/754fb684c7874804b9e34af0d55ed629 is 50, key is test_row_0/B:col10/1695968050348/Put/seqid=0 2023-09-29T06:14:10,913 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/ab44d8773a994b7d89a08e0ff2b0b864 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/ab44d8773a994b7d89a08e0ff2b0b864 2023-09-29T06:14:10,918 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in eefd39c60cf58e105fccf6c634940207/A of eefd39c60cf58e105fccf6c634940207 into ab44d8773a994b7d89a08e0ff2b0b864(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:14:10,919 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:10,919 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207., storeName=eefd39c60cf58e105fccf6c634940207/A, priority=12, startTime=1695968050872; duration=0sec 2023-09-29T06:14:10,919 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:14:10,919 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eefd39c60cf58e105fccf6c634940207:A 2023-09-29T06:14:10,919 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-29T06:14:10,920 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45592 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-29T06:14:10,920 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): eefd39c60cf58e105fccf6c634940207/C is initiating minor compaction (all files) 2023-09-29T06:14:10,920 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of eefd39c60cf58e105fccf6c634940207/C in TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:10,920 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/a559532c42054b8fbd9af3c6e16eb3cf, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/ddedb0a61b1f471ab68d8690326d532e, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/ab63331258dd4ae0ab2f9e03f971b50f, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/929d6588860645fc81eb8427973bcde5] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp, totalSize=44.5 K 2023-09-29T06:14:10,920 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/754fb684c7874804b9e34af0d55ed629 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/754fb684c7874804b9e34af0d55ed629 2023-09-29T06:14:10,921 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting a559532c42054b8fbd9af3c6e16eb3cf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1695968047920 2023-09-29T06:14:10,925 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting ddedb0a61b1f471ab68d8690326d532e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1695968047963 2023-09-29T06:14:10,925 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting ab63331258dd4ae0ab2f9e03f971b50f, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1695968048595 2023-09-29T06:14:10,926 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 929d6588860645fc81eb8427973bcde5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1695968049233 2023-09-29T06:14:10,931 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in eefd39c60cf58e105fccf6c634940207/B of eefd39c60cf58e105fccf6c634940207 into 754fb684c7874804b9e34af0d55ed629(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:14:10,931 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:10,931 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207., storeName=eefd39c60cf58e105fccf6c634940207/B, priority=12, startTime=1695968050872; duration=0sec 2023-09-29T06:14:10,931 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:10,931 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eefd39c60cf58e105fccf6c634940207:B 2023-09-29T06:14:10,936 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eefd39c60cf58e105fccf6c634940207#C#compaction#246 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:14:10,937 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/be616cd901494830b29b778bcf96df45 is 50, key is test_row_0/C:col10/1695968050348/Put/seqid=0 2023-09-29T06:14:10,970 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing eefd39c60cf58e105fccf6c634940207 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2023-09-29T06:14:10,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on eefd39c60cf58e105fccf6c634940207 2023-09-29T06:14:10,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=A 2023-09-29T06:14:10,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:10,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=B 2023-09-29T06:14:10,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:10,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=C 2023-09-29T06:14:10,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:10,982 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/12f9e929b4f847ec9242c9a134ed0054 is 50, key is test_row_0/A:col10/1695968050352/Put/seqid=0 2023-09-29T06:14:10,986 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/be616cd901494830b29b778bcf96df45 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/be616cd901494830b29b778bcf96df45 2023-09-29T06:14:10,994 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in eefd39c60cf58e105fccf6c634940207/C of eefd39c60cf58e105fccf6c634940207 into be616cd901494830b29b778bcf96df45(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:14:10,994 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:10,994 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207., storeName=eefd39c60cf58e105fccf6c634940207/C, priority=12, startTime=1695968050872; duration=0sec 2023-09-29T06:14:10,995 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:10,995 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eefd39c60cf58e105fccf6c634940207:C 2023-09-29T06:14:11,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:11,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968111003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:11,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:11,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968111006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:11,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:11,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968111007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:11,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:11,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968111007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:11,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:11,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968111108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:11,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:11,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968111109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:11,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:11,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968111111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:11,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:11,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968111111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:11,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:11,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968111311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:11,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:11,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968111312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:11,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:11,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968111314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:11,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:11,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968111315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:11,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:11,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41054 deadline: 1695968111358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:11,395 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/12f9e929b4f847ec9242c9a134ed0054 2023-09-29T06:14:11,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/4ae6025b57644dbc9814cabb8431d023 is 50, key is test_row_0/B:col10/1695968050352/Put/seqid=0 2023-09-29T06:14:11,416 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/4ae6025b57644dbc9814cabb8431d023 2023-09-29T06:14:11,427 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/a893683644694f43b8718d37c524f478 is 50, key is test_row_0/C:col10/1695968050352/Put/seqid=0 2023-09-29T06:14:11,451 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/a893683644694f43b8718d37c524f478 2023-09-29T06:14:11,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/12f9e929b4f847ec9242c9a134ed0054 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/12f9e929b4f847ec9242c9a134ed0054 2023-09-29T06:14:11,462 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/12f9e929b4f847ec9242c9a134ed0054, entries=150, sequenceid=95, filesize=11.7 K 2023-09-29T06:14:11,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/4ae6025b57644dbc9814cabb8431d023 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/4ae6025b57644dbc9814cabb8431d023 2023-09-29T06:14:11,468 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/4ae6025b57644dbc9814cabb8431d023, entries=150, sequenceid=95, filesize=11.7 K 2023-09-29T06:14:11,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/a893683644694f43b8718d37c524f478 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/a893683644694f43b8718d37c524f478 2023-09-29T06:14:11,484 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/a893683644694f43b8718d37c524f478, entries=150, sequenceid=95, filesize=11.7 K 2023-09-29T06:14:11,485 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for eefd39c60cf58e105fccf6c634940207 in 515ms, sequenceid=95, compaction requested=false 2023-09-29T06:14:11,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:11,588 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2023-09-29T06:14:11,617 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing eefd39c60cf58e105fccf6c634940207 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2023-09-29T06:14:11,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=A 2023-09-29T06:14:11,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:11,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=B 2023-09-29T06:14:11,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:11,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=C 2023-09-29T06:14:11,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:11,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on eefd39c60cf58e105fccf6c634940207 2023-09-29T06:14:11,628 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/201cc91e94e743d98d401ba2dd5c417b is 50, key is test_row_0/A:col10/1695968051005/Put/seqid=0 2023-09-29T06:14:11,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:11,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968111626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:11,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:11,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968111628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:11,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:11,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968111631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:11,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:11,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968111631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:11,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:11,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968111733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:11,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:11,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968111733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:11,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:11,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968111737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:11,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:11,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968111738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:11,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:11,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968111935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:11,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:11,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968111936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:11,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:11,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968111940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:11,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:11,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968111941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:12,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=73 2023-09-29T06:14:12,011 INFO [Thread-1523 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 73 completed 2023-09-29T06:14:12,013 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:14:12,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=75, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=75, table=TestAcidGuarantees 2023-09-29T06:14:12,017 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=75, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=75, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:14:12,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=75 2023-09-29T06:14:12,017 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=75, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=75, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:14:12,017 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=76, ppid=75, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:14:12,060 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/201cc91e94e743d98d401ba2dd5c417b 2023-09-29T06:14:12,067 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/0d70bd70201840e9a213097087402d45 is 50, key is test_row_0/B:col10/1695968051005/Put/seqid=0 2023-09-29T06:14:12,081 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/0d70bd70201840e9a213097087402d45 2023-09-29T06:14:12,099 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/a88e36fd01674157979701afcc7a0b45 is 50, key is test_row_0/C:col10/1695968051005/Put/seqid=0 2023-09-29T06:14:12,107 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/a88e36fd01674157979701afcc7a0b45 2023-09-29T06:14:12,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/201cc91e94e743d98d401ba2dd5c417b as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/201cc91e94e743d98d401ba2dd5c417b 2023-09-29T06:14:12,117 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/201cc91e94e743d98d401ba2dd5c417b, entries=150, sequenceid=122, filesize=11.7 K 2023-09-29T06:14:12,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=75 2023-09-29T06:14:12,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/0d70bd70201840e9a213097087402d45 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/0d70bd70201840e9a213097087402d45 2023-09-29T06:14:12,123 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/0d70bd70201840e9a213097087402d45, entries=150, sequenceid=122, filesize=11.7 K 2023-09-29T06:14:12,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/a88e36fd01674157979701afcc7a0b45 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/a88e36fd01674157979701afcc7a0b45 2023-09-29T06:14:12,136 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/a88e36fd01674157979701afcc7a0b45, entries=150, sequenceid=122, filesize=11.7 K 2023-09-29T06:14:12,137 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for eefd39c60cf58e105fccf6c634940207 in 520ms, sequenceid=122, compaction requested=true 2023-09-29T06:14:12,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:12,137 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:14:12,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eefd39c60cf58e105fccf6c634940207:A, priority=-2147483648, current under compaction store size is 1 2023-09-29T06:14:12,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:12,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eefd39c60cf58e105fccf6c634940207:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:14:12,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:14:12,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eefd39c60cf58e105fccf6c634940207:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:14:12,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2023-09-29T06:14:12,138 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:14:12,142 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36090 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:14:12,142 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36090 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:14:12,142 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): eefd39c60cf58e105fccf6c634940207/A is initiating minor compaction (all files) 2023-09-29T06:14:12,142 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): eefd39c60cf58e105fccf6c634940207/B is initiating minor compaction (all files) 2023-09-29T06:14:12,142 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of eefd39c60cf58e105fccf6c634940207/A in TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:12,142 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of eefd39c60cf58e105fccf6c634940207/B in TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:12,142 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/ab44d8773a994b7d89a08e0ff2b0b864, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/12f9e929b4f847ec9242c9a134ed0054, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/201cc91e94e743d98d401ba2dd5c417b] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp, totalSize=35.2 K 2023-09-29T06:14:12,142 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/754fb684c7874804b9e34af0d55ed629, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/4ae6025b57644dbc9814cabb8431d023, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/0d70bd70201840e9a213097087402d45] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp, totalSize=35.2 K 2023-09-29T06:14:12,143 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting ab44d8773a994b7d89a08e0ff2b0b864, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1695968049233 2023-09-29T06:14:12,143 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 754fb684c7874804b9e34af0d55ed629, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1695968049233 2023-09-29T06:14:12,144 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 12f9e929b4f847ec9242c9a134ed0054, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1695968050352 2023-09-29T06:14:12,144 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 4ae6025b57644dbc9814cabb8431d023, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1695968050352 2023-09-29T06:14:12,150 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 201cc91e94e743d98d401ba2dd5c417b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1695968051005 2023-09-29T06:14:12,150 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 0d70bd70201840e9a213097087402d45, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1695968051005 2023-09-29T06:14:12,159 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eefd39c60cf58e105fccf6c634940207#A#compaction#253 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:14:12,159 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/73b8f06f0b674f158192d0b7f956275d is 50, key is test_row_0/A:col10/1695968051005/Put/seqid=0 2023-09-29T06:14:12,161 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eefd39c60cf58e105fccf6c634940207#B#compaction#254 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:14:12,161 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/b6ecac7997d6497cb20286229cbb8de9 is 50, key is test_row_0/B:col10/1695968051005/Put/seqid=0 2023-09-29T06:14:12,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=76 2023-09-29T06:14:12,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:12,172 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HRegion(2819): Flushing eefd39c60cf58e105fccf6c634940207 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2023-09-29T06:14:12,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=A 2023-09-29T06:14:12,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:12,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=B 2023-09-29T06:14:12,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:12,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=C 2023-09-29T06:14:12,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:12,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/0df407190a3d4d1abf173167527c8a59 is 50, key is test_row_0/A:col10/1695968051628/Put/seqid=0 2023-09-29T06:14:12,201 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/0df407190a3d4d1abf173167527c8a59 2023-09-29T06:14:12,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/00af6fd8e2a84b2c986646f78867a81b is 50, key is test_row_0/B:col10/1695968051628/Put/seqid=0 2023-09-29T06:14:12,225 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/00af6fd8e2a84b2c986646f78867a81b 2023-09-29T06:14:12,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/fffa3f3a0c42454fb63f52a98bf551d4 is 50, key is test_row_0/C:col10/1695968051628/Put/seqid=0 2023-09-29T06:14:12,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on eefd39c60cf58e105fccf6c634940207 2023-09-29T06:14:12,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. as already flushing 2023-09-29T06:14:12,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:12,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968112275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:12,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:12,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968112274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:12,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:12,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968112276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:12,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:12,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968112277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:12,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=75 2023-09-29T06:14:12,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:12,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968112380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:12,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:12,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:12,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968112380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:12,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968112380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:12,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:12,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968112380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:12,582 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/73b8f06f0b674f158192d0b7f956275d as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/73b8f06f0b674f158192d0b7f956275d 2023-09-29T06:14:12,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:12,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:12,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968112583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:12,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968112584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:12,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:12,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968112585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:12,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:12,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968112585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:12,593 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/b6ecac7997d6497cb20286229cbb8de9 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/b6ecac7997d6497cb20286229cbb8de9 2023-09-29T06:14:12,598 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in eefd39c60cf58e105fccf6c634940207/B of eefd39c60cf58e105fccf6c634940207 into b6ecac7997d6497cb20286229cbb8de9(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:14:12,599 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:12,599 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207., storeName=eefd39c60cf58e105fccf6c634940207/B, priority=13, startTime=1695968052138; duration=0sec 2023-09-29T06:14:12,599 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:14:12,599 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eefd39c60cf58e105fccf6c634940207:B 2023-09-29T06:14:12,599 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:14:12,600 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36090 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:14:12,600 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): eefd39c60cf58e105fccf6c634940207/C is initiating minor compaction (all files) 2023-09-29T06:14:12,600 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of eefd39c60cf58e105fccf6c634940207/C in TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:12,600 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/be616cd901494830b29b778bcf96df45, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/a893683644694f43b8718d37c524f478, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/a88e36fd01674157979701afcc7a0b45] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp, totalSize=35.2 K 2023-09-29T06:14:12,601 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting be616cd901494830b29b778bcf96df45, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1695968049233 2023-09-29T06:14:12,601 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting a893683644694f43b8718d37c524f478, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1695968050352 2023-09-29T06:14:12,602 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting a88e36fd01674157979701afcc7a0b45, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1695968051005 2023-09-29T06:14:12,602 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in eefd39c60cf58e105fccf6c634940207/A of eefd39c60cf58e105fccf6c634940207 into 73b8f06f0b674f158192d0b7f956275d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:14:12,602 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:12,602 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207., storeName=eefd39c60cf58e105fccf6c634940207/A, priority=13, startTime=1695968052137; duration=0sec 2023-09-29T06:14:12,602 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:12,602 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eefd39c60cf58e105fccf6c634940207:A 2023-09-29T06:14:12,610 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eefd39c60cf58e105fccf6c634940207#C#compaction#258 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:14:12,611 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/9aabc69f201946eea45bb68a53123b3b is 50, key is test_row_0/C:col10/1695968051005/Put/seqid=0 2023-09-29T06:14:12,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=75 2023-09-29T06:14:12,620 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/9aabc69f201946eea45bb68a53123b3b as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/9aabc69f201946eea45bb68a53123b3b 2023-09-29T06:14:12,625 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in eefd39c60cf58e105fccf6c634940207/C of eefd39c60cf58e105fccf6c634940207 into 9aabc69f201946eea45bb68a53123b3b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:14:12,625 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:12,625 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207., storeName=eefd39c60cf58e105fccf6c634940207/C, priority=13, startTime=1695968052138; duration=0sec 2023-09-29T06:14:12,625 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:12,625 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eefd39c60cf58e105fccf6c634940207:C 2023-09-29T06:14:12,642 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/fffa3f3a0c42454fb63f52a98bf551d4 2023-09-29T06:14:12,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/0df407190a3d4d1abf173167527c8a59 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/0df407190a3d4d1abf173167527c8a59 2023-09-29T06:14:12,651 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/0df407190a3d4d1abf173167527c8a59, entries=150, sequenceid=132, filesize=11.8 K 2023-09-29T06:14:12,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/00af6fd8e2a84b2c986646f78867a81b as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/00af6fd8e2a84b2c986646f78867a81b 2023-09-29T06:14:12,664 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/00af6fd8e2a84b2c986646f78867a81b, entries=150, sequenceid=132, filesize=11.8 K 2023-09-29T06:14:12,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/fffa3f3a0c42454fb63f52a98bf551d4 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/fffa3f3a0c42454fb63f52a98bf551d4 2023-09-29T06:14:12,671 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/fffa3f3a0c42454fb63f52a98bf551d4, entries=150, sequenceid=132, filesize=11.8 K 2023-09-29T06:14:12,674 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HRegion(3022): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for eefd39c60cf58e105fccf6c634940207 in 502ms, sequenceid=132, compaction requested=false 2023-09-29T06:14:12,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HRegion(2520): Flush status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:12,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:12,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=76 2023-09-29T06:14:12,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=76 2023-09-29T06:14:12,677 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=76, resume processing ppid=75 2023-09-29T06:14:12,677 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=76, ppid=75, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 658 msec 2023-09-29T06:14:12,678 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=75, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=75, table=TestAcidGuarantees in 664 msec 2023-09-29T06:14:12,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on eefd39c60cf58e105fccf6c634940207 2023-09-29T06:14:12,893 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing eefd39c60cf58e105fccf6c634940207 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2023-09-29T06:14:12,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=A 2023-09-29T06:14:12,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:12,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=B 2023-09-29T06:14:12,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:12,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=C 2023-09-29T06:14:12,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:12,900 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/e6fe6cdc2e1b4a7aa67422d67cc886a9 is 50, key is test_row_0/A:col10/1695968052891/Put/seqid=0 2023-09-29T06:14:12,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:12,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968112898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:12,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:12,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968112900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:12,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:12,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968112901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:12,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:12,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968112901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:13,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:13,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968113003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:13,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:13,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968113006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:13,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:13,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968113007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:13,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:13,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968113007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:13,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=75 2023-09-29T06:14:13,121 INFO [Thread-1523 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 75 completed 2023-09-29T06:14:13,122 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:14:13,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=77, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=77, table=TestAcidGuarantees 2023-09-29T06:14:13,125 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=77, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=77, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:14:13,126 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=77, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=77, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:14:13,126 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=78, ppid=77, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:14:13,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=77 2023-09-29T06:14:13,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:13,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968113208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:13,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:13,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:13,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968113209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:13,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968113209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:13,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:13,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968113210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:13,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=77 2023-09-29T06:14:13,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=78 2023-09-29T06:14:13,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:13,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. as already flushing 2023-09-29T06:14:13,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:13,281 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] handler.RSProcedureHandler(53): pid=78 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:13,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=78 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:13,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=78 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:13,320 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/e6fe6cdc2e1b4a7aa67422d67cc886a9 2023-09-29T06:14:13,326 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/da2651d237474a68893c8404ebb0ae4e is 50, key is test_row_0/B:col10/1695968052891/Put/seqid=0 2023-09-29T06:14:13,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:13,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41054 deadline: 1695968113372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:13,379 DEBUG [Thread-1519 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4142 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_2' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207., hostname=jenkins-hbase3.apache.org,39783,1695967978138, seqNum=2, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:572) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:559) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:197) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:218) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:750) 2023-09-29T06:14:13,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=77 2023-09-29T06:14:13,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=78 2023-09-29T06:14:13,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:13,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. as already flushing 2023-09-29T06:14:13,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:13,434 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] handler.RSProcedureHandler(53): pid=78 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:13,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=78 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:13,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=78 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:13,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:13,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968113510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:13,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:13,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968113512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:13,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:13,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968113512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:13,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:13,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968113514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:13,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=78 2023-09-29T06:14:13,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:13,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. as already flushing 2023-09-29T06:14:13,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:13,588 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] handler.RSProcedureHandler(53): pid=78 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:13,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=78 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:13,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=78 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:13,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=77 2023-09-29T06:14:13,740 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=78 2023-09-29T06:14:13,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:13,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. as already flushing 2023-09-29T06:14:13,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:13,740 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] handler.RSProcedureHandler(53): pid=78 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:13,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=78 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:13,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=78 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:13,747 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/da2651d237474a68893c8404ebb0ae4e 2023-09-29T06:14:13,767 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/035bfcf178724ab1bf01d0ccb3c7adb4 is 50, key is test_row_0/C:col10/1695968052891/Put/seqid=0 2023-09-29T06:14:13,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=78 2023-09-29T06:14:13,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:13,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. as already flushing 2023-09-29T06:14:13,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:13,893 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] handler.RSProcedureHandler(53): pid=78 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:13,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=78 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:13,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=78 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:14,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:14,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968114013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:14,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:14,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968114017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:14,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:14,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968114018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:14,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:14,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968114019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:14,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=78 2023-09-29T06:14:14,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:14,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. as already flushing 2023-09-29T06:14:14,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:14,046 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] handler.RSProcedureHandler(53): pid=78 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:14,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=78 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:14,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=78 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:14,196 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/035bfcf178724ab1bf01d0ccb3c7adb4 2023-09-29T06:14:14,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=78 2023-09-29T06:14:14,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:14,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. as already flushing 2023-09-29T06:14:14,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:14,200 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] handler.RSProcedureHandler(53): pid=78 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:14,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=78 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:14,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/e6fe6cdc2e1b4a7aa67422d67cc886a9 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/e6fe6cdc2e1b4a7aa67422d67cc886a9 2023-09-29T06:14:14,205 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/e6fe6cdc2e1b4a7aa67422d67cc886a9, entries=150, sequenceid=162, filesize=11.8 K 2023-09-29T06:14:14,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=78 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:14,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/da2651d237474a68893c8404ebb0ae4e as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/da2651d237474a68893c8404ebb0ae4e 2023-09-29T06:14:14,213 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/da2651d237474a68893c8404ebb0ae4e, entries=150, sequenceid=162, filesize=11.8 K 2023-09-29T06:14:14,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/035bfcf178724ab1bf01d0ccb3c7adb4 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/035bfcf178724ab1bf01d0ccb3c7adb4 2023-09-29T06:14:14,228 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/035bfcf178724ab1bf01d0ccb3c7adb4, entries=150, sequenceid=162, filesize=11.8 K 2023-09-29T06:14:14,229 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for eefd39c60cf58e105fccf6c634940207 in 1337ms, sequenceid=162, compaction requested=true 2023-09-29T06:14:14,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:14,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eefd39c60cf58e105fccf6c634940207:A, priority=-2147483648, current under compaction store size is 1 2023-09-29T06:14:14,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:14,229 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:14:14,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eefd39c60cf58e105fccf6c634940207:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:14:14,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:14,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eefd39c60cf58e105fccf6c634940207:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:14:14,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:14:14,230 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:14:14,231 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36492 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:14:14,231 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): eefd39c60cf58e105fccf6c634940207/A is initiating minor compaction (all files) 2023-09-29T06:14:14,231 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of eefd39c60cf58e105fccf6c634940207/A in TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:14,231 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/73b8f06f0b674f158192d0b7f956275d, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/0df407190a3d4d1abf173167527c8a59, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/e6fe6cdc2e1b4a7aa67422d67cc886a9] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp, totalSize=35.6 K 2023-09-29T06:14:14,232 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 73b8f06f0b674f158192d0b7f956275d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1695968051005 2023-09-29T06:14:14,232 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36492 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:14:14,232 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): eefd39c60cf58e105fccf6c634940207/B is initiating minor compaction (all files) 2023-09-29T06:14:14,232 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of eefd39c60cf58e105fccf6c634940207/B in TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:14,232 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 0df407190a3d4d1abf173167527c8a59, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1695968051626 2023-09-29T06:14:14,232 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/b6ecac7997d6497cb20286229cbb8de9, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/00af6fd8e2a84b2c986646f78867a81b, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/da2651d237474a68893c8404ebb0ae4e] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp, totalSize=35.6 K 2023-09-29T06:14:14,232 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting e6fe6cdc2e1b4a7aa67422d67cc886a9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1695968052272 2023-09-29T06:14:14,233 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting b6ecac7997d6497cb20286229cbb8de9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1695968051005 2023-09-29T06:14:14,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=77 2023-09-29T06:14:14,233 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 00af6fd8e2a84b2c986646f78867a81b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1695968051626 2023-09-29T06:14:14,234 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting da2651d237474a68893c8404ebb0ae4e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1695968052272 2023-09-29T06:14:14,241 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eefd39c60cf58e105fccf6c634940207#A#compaction#262 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:14:14,241 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/e1d7734c00084a869e61c9fee9a14a7c is 50, key is test_row_0/A:col10/1695968052891/Put/seqid=0 2023-09-29T06:14:14,244 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eefd39c60cf58e105fccf6c634940207#B#compaction#263 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:14:14,245 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/ab3f417decf24aa5aa8e5e1eea38de35 is 50, key is test_row_0/B:col10/1695968052891/Put/seqid=0 2023-09-29T06:14:14,252 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/e1d7734c00084a869e61c9fee9a14a7c as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/e1d7734c00084a869e61c9fee9a14a7c 2023-09-29T06:14:14,257 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in eefd39c60cf58e105fccf6c634940207/A of eefd39c60cf58e105fccf6c634940207 into e1d7734c00084a869e61c9fee9a14a7c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:14:14,257 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:14,257 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207., storeName=eefd39c60cf58e105fccf6c634940207/A, priority=13, startTime=1695968054229; duration=0sec 2023-09-29T06:14:14,257 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:14:14,258 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eefd39c60cf58e105fccf6c634940207:A 2023-09-29T06:14:14,258 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:14:14,260 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36492 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:14:14,260 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): eefd39c60cf58e105fccf6c634940207/C is initiating minor compaction (all files) 2023-09-29T06:14:14,260 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of eefd39c60cf58e105fccf6c634940207/C in TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:14,260 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/9aabc69f201946eea45bb68a53123b3b, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/fffa3f3a0c42454fb63f52a98bf551d4, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/035bfcf178724ab1bf01d0ccb3c7adb4] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp, totalSize=35.6 K 2023-09-29T06:14:14,261 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 9aabc69f201946eea45bb68a53123b3b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1695968051005 2023-09-29T06:14:14,261 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting fffa3f3a0c42454fb63f52a98bf551d4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1695968051626 2023-09-29T06:14:14,262 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 035bfcf178724ab1bf01d0ccb3c7adb4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1695968052272 2023-09-29T06:14:14,279 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/ab3f417decf24aa5aa8e5e1eea38de35 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/ab3f417decf24aa5aa8e5e1eea38de35 2023-09-29T06:14:14,288 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in eefd39c60cf58e105fccf6c634940207/B of eefd39c60cf58e105fccf6c634940207 into ab3f417decf24aa5aa8e5e1eea38de35(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:14:14,288 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:14,288 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207., storeName=eefd39c60cf58e105fccf6c634940207/B, priority=13, startTime=1695968054229; duration=0sec 2023-09-29T06:14:14,288 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:14,288 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eefd39c60cf58e105fccf6c634940207:B 2023-09-29T06:14:14,304 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eefd39c60cf58e105fccf6c634940207#C#compaction#264 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:14:14,305 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/4d73fdb1bd9d494e973f85dedd3a16f6 is 50, key is test_row_0/C:col10/1695968052891/Put/seqid=0 2023-09-29T06:14:14,320 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/4d73fdb1bd9d494e973f85dedd3a16f6 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/4d73fdb1bd9d494e973f85dedd3a16f6 2023-09-29T06:14:14,328 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in eefd39c60cf58e105fccf6c634940207/C of eefd39c60cf58e105fccf6c634940207 into 4d73fdb1bd9d494e973f85dedd3a16f6(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:14:14,328 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:14,328 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207., storeName=eefd39c60cf58e105fccf6c634940207/C, priority=13, startTime=1695968054230; duration=0sec 2023-09-29T06:14:14,328 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:14,328 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eefd39c60cf58e105fccf6c634940207:C 2023-09-29T06:14:14,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=78 2023-09-29T06:14:14,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:14,357 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegion(2819): Flushing eefd39c60cf58e105fccf6c634940207 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2023-09-29T06:14:14,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=A 2023-09-29T06:14:14,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:14,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=B 2023-09-29T06:14:14,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:14,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=C 2023-09-29T06:14:14,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:14,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/b4f0c48fb5c7498388cbf2890705e1d6 is 50, key is test_row_0/A:col10/1695968052899/Put/seqid=0 2023-09-29T06:14:14,768 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/b4f0c48fb5c7498388cbf2890705e1d6 2023-09-29T06:14:14,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/c18bf2fb56ba47008ea38720958940f2 is 50, key is test_row_0/B:col10/1695968052899/Put/seqid=0 2023-09-29T06:14:15,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on eefd39c60cf58e105fccf6c634940207 2023-09-29T06:14:15,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. as already flushing 2023-09-29T06:14:15,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:15,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968115068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:15,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:15,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968115071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:15,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:15,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968115073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:15,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:15,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968115075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:15,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:15,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968115173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:15,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:15,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968115176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:15,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:15,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968115177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:15,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:15,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968115178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:15,182 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/c18bf2fb56ba47008ea38720958940f2 2023-09-29T06:14:15,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/b3a0352185ea429a8c3127ec6ee75e61 is 50, key is test_row_0/C:col10/1695968052899/Put/seqid=0 2023-09-29T06:14:15,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=77 2023-09-29T06:14:15,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:15,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968115377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:15,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:15,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968115379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:15,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:15,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:15,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968115379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:15,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968115379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:15,621 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/b3a0352185ea429a8c3127ec6ee75e61 2023-09-29T06:14:15,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/b4f0c48fb5c7498388cbf2890705e1d6 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/b4f0c48fb5c7498388cbf2890705e1d6 2023-09-29T06:14:15,632 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/b4f0c48fb5c7498388cbf2890705e1d6, entries=150, sequenceid=175, filesize=11.8 K 2023-09-29T06:14:15,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/c18bf2fb56ba47008ea38720958940f2 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/c18bf2fb56ba47008ea38720958940f2 2023-09-29T06:14:15,637 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/c18bf2fb56ba47008ea38720958940f2, entries=150, sequenceid=175, filesize=11.8 K 2023-09-29T06:14:15,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/b3a0352185ea429a8c3127ec6ee75e61 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/b3a0352185ea429a8c3127ec6ee75e61 2023-09-29T06:14:15,645 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/b3a0352185ea429a8c3127ec6ee75e61, entries=150, sequenceid=175, filesize=11.8 K 2023-09-29T06:14:15,655 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegion(3022): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=161.02 KB/164880 for eefd39c60cf58e105fccf6c634940207 in 1298ms, sequenceid=175, compaction requested=false 2023-09-29T06:14:15,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegion(2520): Flush status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:15,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:15,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=78 2023-09-29T06:14:15,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=78 2023-09-29T06:14:15,658 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=78, resume processing ppid=77 2023-09-29T06:14:15,659 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=78, ppid=77, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5310 sec 2023-09-29T06:14:15,661 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=77, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=77, table=TestAcidGuarantees in 2.5370 sec 2023-09-29T06:14:15,683 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing eefd39c60cf58e105fccf6c634940207 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2023-09-29T06:14:15,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=A 2023-09-29T06:14:15,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:15,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=B 2023-09-29T06:14:15,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:15,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=C 2023-09-29T06:14:15,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:15,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on eefd39c60cf58e105fccf6c634940207 2023-09-29T06:14:15,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/ddf5c3cb24ea46c1b18c5bea49ba4992 is 50, key is test_row_0/A:col10/1695968055682/Put/seqid=0 2023-09-29T06:14:15,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:15,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968115687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:15,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:15,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968115689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:15,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:15,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968115690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:15,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:15,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968115691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:15,705 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/ddf5c3cb24ea46c1b18c5bea49ba4992 2023-09-29T06:14:15,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/b0f024219dba4e3f87393c13c7eff821 is 50, key is test_row_0/B:col10/1695968055682/Put/seqid=0 2023-09-29T06:14:15,722 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/b0f024219dba4e3f87393c13c7eff821 2023-09-29T06:14:15,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/919f04a5c7b247868e1f380e2a9637b5 is 50, key is test_row_0/C:col10/1695968055682/Put/seqid=0 2023-09-29T06:14:15,741 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/919f04a5c7b247868e1f380e2a9637b5 2023-09-29T06:14:15,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/ddf5c3cb24ea46c1b18c5bea49ba4992 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/ddf5c3cb24ea46c1b18c5bea49ba4992 2023-09-29T06:14:15,753 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/ddf5c3cb24ea46c1b18c5bea49ba4992, entries=150, sequenceid=203, filesize=11.8 K 2023-09-29T06:14:15,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/b0f024219dba4e3f87393c13c7eff821 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/b0f024219dba4e3f87393c13c7eff821 2023-09-29T06:14:15,758 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/b0f024219dba4e3f87393c13c7eff821, entries=150, sequenceid=203, filesize=11.8 K 2023-09-29T06:14:15,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/919f04a5c7b247868e1f380e2a9637b5 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/919f04a5c7b247868e1f380e2a9637b5 2023-09-29T06:14:15,763 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/919f04a5c7b247868e1f380e2a9637b5, entries=150, sequenceid=203, filesize=11.8 K 2023-09-29T06:14:15,763 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for eefd39c60cf58e105fccf6c634940207 in 80ms, sequenceid=203, compaction requested=true 2023-09-29T06:14:15,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:15,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eefd39c60cf58e105fccf6c634940207:A, priority=-2147483648, current under compaction store size is 1 2023-09-29T06:14:15,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:15,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eefd39c60cf58e105fccf6c634940207:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:14:15,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:15,764 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:14:15,764 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:14:15,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eefd39c60cf58e105fccf6c634940207:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:14:15,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:14:15,765 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36744 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:14:15,765 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): eefd39c60cf58e105fccf6c634940207/B is initiating minor compaction (all files) 2023-09-29T06:14:15,765 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of eefd39c60cf58e105fccf6c634940207/B in TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:15,765 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/ab3f417decf24aa5aa8e5e1eea38de35, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/c18bf2fb56ba47008ea38720958940f2, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/b0f024219dba4e3f87393c13c7eff821] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp, totalSize=35.9 K 2023-09-29T06:14:15,765 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36744 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:14:15,765 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting ab3f417decf24aa5aa8e5e1eea38de35, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1695968052272 2023-09-29T06:14:15,765 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): eefd39c60cf58e105fccf6c634940207/A is initiating minor compaction (all files) 2023-09-29T06:14:15,766 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of eefd39c60cf58e105fccf6c634940207/A in TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:15,766 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/e1d7734c00084a869e61c9fee9a14a7c, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/b4f0c48fb5c7498388cbf2890705e1d6, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/ddf5c3cb24ea46c1b18c5bea49ba4992] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp, totalSize=35.9 K 2023-09-29T06:14:15,766 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting c18bf2fb56ba47008ea38720958940f2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1695968052898 2023-09-29T06:14:15,766 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting e1d7734c00084a869e61c9fee9a14a7c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1695968052272 2023-09-29T06:14:15,767 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting b0f024219dba4e3f87393c13c7eff821, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1695968055070 2023-09-29T06:14:15,767 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting b4f0c48fb5c7498388cbf2890705e1d6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1695968052898 2023-09-29T06:14:15,767 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting ddf5c3cb24ea46c1b18c5bea49ba4992, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1695968055070 2023-09-29T06:14:15,790 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eefd39c60cf58e105fccf6c634940207#B#compaction#271 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:14:15,791 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/ccb6010a791f43b0a6f4f796cc4065ed is 50, key is test_row_0/B:col10/1695968055682/Put/seqid=0 2023-09-29T06:14:15,791 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eefd39c60cf58e105fccf6c634940207#A#compaction#272 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:14:15,792 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/0f80d17246284443b1c8d291ab8bfcb7 is 50, key is test_row_0/A:col10/1695968055682/Put/seqid=0 2023-09-29T06:14:15,797 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing eefd39c60cf58e105fccf6c634940207 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2023-09-29T06:14:15,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on eefd39c60cf58e105fccf6c634940207 2023-09-29T06:14:15,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=A 2023-09-29T06:14:15,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:15,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=B 2023-09-29T06:14:15,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:15,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=C 2023-09-29T06:14:15,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:15,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/813f62168c4346d4995082ed0697d22c is 50, key is test_row_0/A:col10/1695968055796/Put/seqid=0 2023-09-29T06:14:15,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:15,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968115828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:15,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:15,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968115830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:15,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:15,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968115830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:15,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:15,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968115830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:15,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:15,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968115933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:15,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:15,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968115935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:15,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:15,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968115935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:15,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:15,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968115935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:16,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:16,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968116137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:16,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:16,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968116139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:16,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:16,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968116146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:16,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:16,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968116146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:16,203 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/ccb6010a791f43b0a6f4f796cc4065ed as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/ccb6010a791f43b0a6f4f796cc4065ed 2023-09-29T06:14:16,207 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in eefd39c60cf58e105fccf6c634940207/B of eefd39c60cf58e105fccf6c634940207 into ccb6010a791f43b0a6f4f796cc4065ed(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:14:16,207 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:16,207 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207., storeName=eefd39c60cf58e105fccf6c634940207/B, priority=13, startTime=1695968055764; duration=0sec 2023-09-29T06:14:16,208 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:14:16,208 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eefd39c60cf58e105fccf6c634940207:B 2023-09-29T06:14:16,208 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:14:16,209 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36744 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:14:16,209 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): eefd39c60cf58e105fccf6c634940207/C is initiating minor compaction (all files) 2023-09-29T06:14:16,210 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of eefd39c60cf58e105fccf6c634940207/C in TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:16,210 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/4d73fdb1bd9d494e973f85dedd3a16f6, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/b3a0352185ea429a8c3127ec6ee75e61, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/919f04a5c7b247868e1f380e2a9637b5] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp, totalSize=35.9 K 2023-09-29T06:14:16,210 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 4d73fdb1bd9d494e973f85dedd3a16f6, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1695968052272 2023-09-29T06:14:16,211 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting b3a0352185ea429a8c3127ec6ee75e61, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1695968052898 2023-09-29T06:14:16,212 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 919f04a5c7b247868e1f380e2a9637b5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1695968055070 2023-09-29T06:14:16,220 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eefd39c60cf58e105fccf6c634940207#C#compaction#274 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:14:16,221 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/d198ee62a70e444ea62722d5fc9d83c8 is 50, key is test_row_0/C:col10/1695968055682/Put/seqid=0 2023-09-29T06:14:16,249 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/0f80d17246284443b1c8d291ab8bfcb7 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/0f80d17246284443b1c8d291ab8bfcb7 2023-09-29T06:14:16,253 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/813f62168c4346d4995082ed0697d22c 2023-09-29T06:14:16,255 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in eefd39c60cf58e105fccf6c634940207/A of eefd39c60cf58e105fccf6c634940207 into 0f80d17246284443b1c8d291ab8bfcb7(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:14:16,255 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:16,255 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207., storeName=eefd39c60cf58e105fccf6c634940207/A, priority=13, startTime=1695968055764; duration=0sec 2023-09-29T06:14:16,255 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:16,255 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eefd39c60cf58e105fccf6c634940207:A 2023-09-29T06:14:16,260 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/7678041864c84240bd4cf5211970d90c is 50, key is test_row_0/B:col10/1695968055796/Put/seqid=0 2023-09-29T06:14:16,280 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/7678041864c84240bd4cf5211970d90c 2023-09-29T06:14:16,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/d675812fb4de44f9a9c69972a6b24378 is 50, key is test_row_0/C:col10/1695968055796/Put/seqid=0 2023-09-29T06:14:16,294 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/d675812fb4de44f9a9c69972a6b24378 2023-09-29T06:14:16,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/813f62168c4346d4995082ed0697d22c as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/813f62168c4346d4995082ed0697d22c 2023-09-29T06:14:16,304 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/813f62168c4346d4995082ed0697d22c, entries=200, sequenceid=216, filesize=14.2 K 2023-09-29T06:14:16,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/7678041864c84240bd4cf5211970d90c as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/7678041864c84240bd4cf5211970d90c 2023-09-29T06:14:16,309 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/7678041864c84240bd4cf5211970d90c, entries=150, sequenceid=216, filesize=11.8 K 2023-09-29T06:14:16,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/d675812fb4de44f9a9c69972a6b24378 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/d675812fb4de44f9a9c69972a6b24378 2023-09-29T06:14:16,316 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/d675812fb4de44f9a9c69972a6b24378, entries=150, sequenceid=216, filesize=11.8 K 2023-09-29T06:14:16,317 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=147.60 KB/151140 for eefd39c60cf58e105fccf6c634940207 in 520ms, sequenceid=216, compaction requested=false 2023-09-29T06:14:16,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:16,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on eefd39c60cf58e105fccf6c634940207 2023-09-29T06:14:16,446 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing eefd39c60cf58e105fccf6c634940207 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2023-09-29T06:14:16,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=A 2023-09-29T06:14:16,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:16,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=B 2023-09-29T06:14:16,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:16,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=C 2023-09-29T06:14:16,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:16,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/6d5cca0f542c4d87869e67aaf86935f5 is 50, key is test_row_0/A:col10/1695968055821/Put/seqid=0 2023-09-29T06:14:16,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:16,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968116455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:16,458 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/6d5cca0f542c4d87869e67aaf86935f5 2023-09-29T06:14:16,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:16,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968116457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:16,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:16,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968116458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:16,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:16,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968116462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:16,469 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/6f5e0bb75c1247eabf6f2421cfd210ce is 50, key is test_row_0/B:col10/1695968055821/Put/seqid=0 2023-09-29T06:14:16,491 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/6f5e0bb75c1247eabf6f2421cfd210ce 2023-09-29T06:14:16,501 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/b8166efea80040b8b84081478e84a5f1 is 50, key is test_row_0/C:col10/1695968055821/Put/seqid=0 2023-09-29T06:14:16,510 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/b8166efea80040b8b84081478e84a5f1 2023-09-29T06:14:16,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/6d5cca0f542c4d87869e67aaf86935f5 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/6d5cca0f542c4d87869e67aaf86935f5 2023-09-29T06:14:16,519 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/6d5cca0f542c4d87869e67aaf86935f5, entries=150, sequenceid=244, filesize=11.8 K 2023-09-29T06:14:16,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/6f5e0bb75c1247eabf6f2421cfd210ce as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/6f5e0bb75c1247eabf6f2421cfd210ce 2023-09-29T06:14:16,527 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/6f5e0bb75c1247eabf6f2421cfd210ce, entries=150, sequenceid=244, filesize=11.8 K 2023-09-29T06:14:16,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/b8166efea80040b8b84081478e84a5f1 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/b8166efea80040b8b84081478e84a5f1 2023-09-29T06:14:16,531 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/b8166efea80040b8b84081478e84a5f1, entries=150, sequenceid=244, filesize=11.8 K 2023-09-29T06:14:16,532 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=60.38 KB/61830 for eefd39c60cf58e105fccf6c634940207 in 86ms, sequenceid=244, compaction requested=true 2023-09-29T06:14:16,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:16,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eefd39c60cf58e105fccf6c634940207:A, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:14:16,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:16,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eefd39c60cf58e105fccf6c634940207:B, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:14:16,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:14:16,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eefd39c60cf58e105fccf6c634940207:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:14:16,532 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:14:16,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2023-09-29T06:14:16,533 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39236 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:14:16,533 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): eefd39c60cf58e105fccf6c634940207/A is initiating minor compaction (all files) 2023-09-29T06:14:16,533 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of eefd39c60cf58e105fccf6c634940207/A in TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:16,534 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/0f80d17246284443b1c8d291ab8bfcb7, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/813f62168c4346d4995082ed0697d22c, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/6d5cca0f542c4d87869e67aaf86935f5] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp, totalSize=38.3 K 2023-09-29T06:14:16,534 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 0f80d17246284443b1c8d291ab8bfcb7, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1695968055070 2023-09-29T06:14:16,535 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 813f62168c4346d4995082ed0697d22c, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1695968055689 2023-09-29T06:14:16,536 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 6d5cca0f542c4d87869e67aaf86935f5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1695968055821 2023-09-29T06:14:16,553 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eefd39c60cf58e105fccf6c634940207#A#compaction#280 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:14:16,553 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/f4742bf9ffa14c61be6959892aad810a is 50, key is test_row_0/A:col10/1695968055821/Put/seqid=0 2023-09-29T06:14:16,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on eefd39c60cf58e105fccf6c634940207 2023-09-29T06:14:16,561 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing eefd39c60cf58e105fccf6c634940207 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2023-09-29T06:14:16,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=A 2023-09-29T06:14:16,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:16,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=B 2023-09-29T06:14:16,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:16,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=C 2023-09-29T06:14:16,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:16,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/add58ffd1a3348979c943c3c2aee8dce is 50, key is test_row_0/A:col10/1695968056457/Put/seqid=0 2023-09-29T06:14:16,568 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/f4742bf9ffa14c61be6959892aad810a as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/f4742bf9ffa14c61be6959892aad810a 2023-09-29T06:14:16,574 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in eefd39c60cf58e105fccf6c634940207/A of eefd39c60cf58e105fccf6c634940207 into f4742bf9ffa14c61be6959892aad810a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:14:16,574 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:16,574 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207., storeName=eefd39c60cf58e105fccf6c634940207/A, priority=13, startTime=1695968056532; duration=0sec 2023-09-29T06:14:16,574 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2023-09-29T06:14:16,574 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eefd39c60cf58e105fccf6c634940207:A 2023-09-29T06:14:16,574 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 3 compacting, 2 eligible, 16 blocking 2023-09-29T06:14:16,578 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2023-09-29T06:14:16,578 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2023-09-29T06:14:16,578 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. because compaction request was cancelled 2023-09-29T06:14:16,578 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eefd39c60cf58e105fccf6c634940207:C 2023-09-29T06:14:16,578 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:14:16,582 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/add58ffd1a3348979c943c3c2aee8dce 2023-09-29T06:14:16,584 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36846 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:14:16,584 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): eefd39c60cf58e105fccf6c634940207/B is initiating minor compaction (all files) 2023-09-29T06:14:16,585 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of eefd39c60cf58e105fccf6c634940207/B in TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:16,585 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/ccb6010a791f43b0a6f4f796cc4065ed, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/7678041864c84240bd4cf5211970d90c, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/6f5e0bb75c1247eabf6f2421cfd210ce] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp, totalSize=36.0 K 2023-09-29T06:14:16,585 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting ccb6010a791f43b0a6f4f796cc4065ed, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1695968055070 2023-09-29T06:14:16,586 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 7678041864c84240bd4cf5211970d90c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1695968055794 2023-09-29T06:14:16,586 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 6f5e0bb75c1247eabf6f2421cfd210ce, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1695968055821 2023-09-29T06:14:16,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/93478b8d68cf4298b16dd9d82a666f44 is 50, key is test_row_0/B:col10/1695968056457/Put/seqid=0 2023-09-29T06:14:16,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:16,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968116602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:16,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:16,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968116603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:16,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:16,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968116604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:16,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:16,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968116606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:16,611 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eefd39c60cf58e105fccf6c634940207#B#compaction#283 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:14:16,612 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/4bc55da3d3ac43049556b842a22ef33c is 50, key is test_row_0/B:col10/1695968055821/Put/seqid=0 2023-09-29T06:14:16,616 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/93478b8d68cf4298b16dd9d82a666f44 2023-09-29T06:14:16,624 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/4bc55da3d3ac43049556b842a22ef33c as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/4bc55da3d3ac43049556b842a22ef33c 2023-09-29T06:14:16,626 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/c99c5b538c854f74a7c7146a5c41a727 is 50, key is test_row_0/C:col10/1695968056457/Put/seqid=0 2023-09-29T06:14:16,635 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/c99c5b538c854f74a7c7146a5c41a727 2023-09-29T06:14:16,636 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in eefd39c60cf58e105fccf6c634940207/B of eefd39c60cf58e105fccf6c634940207 into 4bc55da3d3ac43049556b842a22ef33c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:14:16,636 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:16,636 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207., storeName=eefd39c60cf58e105fccf6c634940207/B, priority=13, startTime=1695968056532; duration=0sec 2023-09-29T06:14:16,636 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:16,636 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eefd39c60cf58e105fccf6c634940207:B 2023-09-29T06:14:16,638 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/d198ee62a70e444ea62722d5fc9d83c8 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/d198ee62a70e444ea62722d5fc9d83c8 2023-09-29T06:14:16,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/add58ffd1a3348979c943c3c2aee8dce as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/add58ffd1a3348979c943c3c2aee8dce 2023-09-29T06:14:16,644 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in eefd39c60cf58e105fccf6c634940207/C of eefd39c60cf58e105fccf6c634940207 into d198ee62a70e444ea62722d5fc9d83c8(size=12.3 K), total size for store is 36.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:14:16,644 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:16,644 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207., storeName=eefd39c60cf58e105fccf6c634940207/C, priority=13, startTime=1695968055764; duration=0sec 2023-09-29T06:14:16,644 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:16,644 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eefd39c60cf58e105fccf6c634940207:C 2023-09-29T06:14:16,646 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/add58ffd1a3348979c943c3c2aee8dce, entries=200, sequenceid=257, filesize=14.2 K 2023-09-29T06:14:16,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/93478b8d68cf4298b16dd9d82a666f44 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/93478b8d68cf4298b16dd9d82a666f44 2023-09-29T06:14:16,650 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/93478b8d68cf4298b16dd9d82a666f44, entries=150, sequenceid=257, filesize=11.9 K 2023-09-29T06:14:16,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/c99c5b538c854f74a7c7146a5c41a727 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/c99c5b538c854f74a7c7146a5c41a727 2023-09-29T06:14:16,655 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/c99c5b538c854f74a7c7146a5c41a727, entries=150, sequenceid=257, filesize=11.9 K 2023-09-29T06:14:16,656 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for eefd39c60cf58e105fccf6c634940207 in 95ms, sequenceid=257, compaction requested=true 2023-09-29T06:14:16,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:16,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eefd39c60cf58e105fccf6c634940207:A, priority=-2147483648, current under compaction store size is 1 2023-09-29T06:14:16,656 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2023-09-29T06:14:16,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:16,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eefd39c60cf58e105fccf6c634940207:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:14:16,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:16,656 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2023-09-29T06:14:16,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eefd39c60cf58e105fccf6c634940207:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:14:16,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:14:16,657 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2023-09-29T06:14:16,657 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2023-09-29T06:14:16,657 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. because compaction request was cancelled 2023-09-29T06:14:16,657 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eefd39c60cf58e105fccf6c634940207:A 2023-09-29T06:14:16,657 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-29T06:14:16,657 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2023-09-29T06:14:16,657 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2023-09-29T06:14:16,657 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. because compaction request was cancelled 2023-09-29T06:14:16,657 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eefd39c60cf58e105fccf6c634940207:B 2023-09-29T06:14:16,658 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49030 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-29T06:14:16,658 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): eefd39c60cf58e105fccf6c634940207/C is initiating minor compaction (all files) 2023-09-29T06:14:16,658 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of eefd39c60cf58e105fccf6c634940207/C in TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:16,658 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/d198ee62a70e444ea62722d5fc9d83c8, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/d675812fb4de44f9a9c69972a6b24378, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/b8166efea80040b8b84081478e84a5f1, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/c99c5b538c854f74a7c7146a5c41a727] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp, totalSize=47.9 K 2023-09-29T06:14:16,659 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting d198ee62a70e444ea62722d5fc9d83c8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1695968055070 2023-09-29T06:14:16,659 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting d675812fb4de44f9a9c69972a6b24378, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1695968055794 2023-09-29T06:14:16,660 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting b8166efea80040b8b84081478e84a5f1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1695968055821 2023-09-29T06:14:16,660 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting c99c5b538c854f74a7c7146a5c41a727, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1695968056456 2023-09-29T06:14:16,680 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eefd39c60cf58e105fccf6c634940207#C#compaction#285 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:14:16,680 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/e4ddcef096f1475594dd4ce155f80fee is 50, key is test_row_0/C:col10/1695968056457/Put/seqid=0 2023-09-29T06:14:16,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on eefd39c60cf58e105fccf6c634940207 2023-09-29T06:14:16,709 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing eefd39c60cf58e105fccf6c634940207 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2023-09-29T06:14:16,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=A 2023-09-29T06:14:16,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:16,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=B 2023-09-29T06:14:16,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:16,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=C 2023-09-29T06:14:16,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:16,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/f6ab128c2a5043efa34f082436768f34 is 50, key is test_row_0/A:col10/1695968056709/Put/seqid=0 2023-09-29T06:14:16,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:16,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968116718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:16,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:16,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968116720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:16,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:16,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968116720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:16,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:16,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968116721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:16,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:16,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968116824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:16,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:16,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968116824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:16,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:16,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968116824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:16,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:16,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968116824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:17,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:17,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968117026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:17,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:17,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968117026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:17,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:17,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968117034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:17,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:17,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968117034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:17,104 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/e4ddcef096f1475594dd4ce155f80fee as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/e4ddcef096f1475594dd4ce155f80fee 2023-09-29T06:14:17,109 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in eefd39c60cf58e105fccf6c634940207/C of eefd39c60cf58e105fccf6c634940207 into e4ddcef096f1475594dd4ce155f80fee(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:14:17,109 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:17,109 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207., storeName=eefd39c60cf58e105fccf6c634940207/C, priority=12, startTime=1695968056656; duration=0sec 2023-09-29T06:14:17,109 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:17,109 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eefd39c60cf58e105fccf6c634940207:C 2023-09-29T06:14:17,136 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/f6ab128c2a5043efa34f082436768f34 2023-09-29T06:14:17,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/b5a3035d5a08424b9319f18f231469ea is 50, key is test_row_0/B:col10/1695968056709/Put/seqid=0 2023-09-29T06:14:17,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=77 2023-09-29T06:14:17,235 INFO [Thread-1523 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 77 completed 2023-09-29T06:14:17,237 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:14:17,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=79, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=79, table=TestAcidGuarantees 2023-09-29T06:14:17,244 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=79, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=79, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:14:17,245 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=79, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=79, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:14:17,245 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:14:17,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=79 2023-09-29T06:14:17,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:17,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968117331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:17,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:17,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968117331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:17,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:17,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968117337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:17,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:17,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968117337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:17,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=79 2023-09-29T06:14:17,402 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=80 2023-09-29T06:14:17,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:17,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. as already flushing 2023-09-29T06:14:17,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:17,402 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] handler.RSProcedureHandler(53): pid=80 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:17,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=80 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:17,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=80 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:17,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:17,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41054 deadline: 1695968117404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:17,406 DEBUG [Thread-1519 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=7, retries=16, started=8173 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_2' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207., hostname=jenkins-hbase3.apache.org,39783,1695967978138, seqNum=2, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:572) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:559) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:197) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:218) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:750) 2023-09-29T06:14:17,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=79 2023-09-29T06:14:17,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=80 2023-09-29T06:14:17,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:17,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. as already flushing 2023-09-29T06:14:17,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:17,555 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=80}] handler.RSProcedureHandler(53): pid=80 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:17,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=80 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:17,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=80 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:17,561 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/b5a3035d5a08424b9319f18f231469ea 2023-09-29T06:14:17,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/841e9b2f5e094b4c9f5dec742923aaeb is 50, key is test_row_0/C:col10/1695968056709/Put/seqid=0 2023-09-29T06:14:17,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=80 2023-09-29T06:14:17,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:17,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. as already flushing 2023-09-29T06:14:17,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:17,708 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] handler.RSProcedureHandler(53): pid=80 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:17,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=80 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:17,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=80 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:17,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:17,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968117835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:17,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:17,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968117838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:17,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:17,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968117841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:17,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:17,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968117841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:17,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=79 2023-09-29T06:14:17,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=80 2023-09-29T06:14:17,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:17,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. as already flushing 2023-09-29T06:14:17,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:17,861 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] handler.RSProcedureHandler(53): pid=80 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:17,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=80 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:17,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=80 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:17,992 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/841e9b2f5e094b4c9f5dec742923aaeb 2023-09-29T06:14:17,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/f6ab128c2a5043efa34f082436768f34 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/f6ab128c2a5043efa34f082436768f34 2023-09-29T06:14:18,012 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/f6ab128c2a5043efa34f082436768f34, entries=200, sequenceid=286, filesize=14.4 K 2023-09-29T06:14:18,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/b5a3035d5a08424b9319f18f231469ea as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/b5a3035d5a08424b9319f18f231469ea 2023-09-29T06:14:18,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=80 2023-09-29T06:14:18,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:18,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. as already flushing 2023-09-29T06:14:18,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:18,014 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=80}] handler.RSProcedureHandler(53): pid=80 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:18,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=80 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:18,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=80 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:18,031 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/b5a3035d5a08424b9319f18f231469ea, entries=150, sequenceid=286, filesize=12.0 K 2023-09-29T06:14:18,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/841e9b2f5e094b4c9f5dec742923aaeb as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/841e9b2f5e094b4c9f5dec742923aaeb 2023-09-29T06:14:18,037 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/841e9b2f5e094b4c9f5dec742923aaeb, entries=150, sequenceid=286, filesize=12.0 K 2023-09-29T06:14:18,038 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for eefd39c60cf58e105fccf6c634940207 in 1329ms, sequenceid=286, compaction requested=true 2023-09-29T06:14:18,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:18,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eefd39c60cf58e105fccf6c634940207:A, priority=-2147483648, current under compaction store size is 1 2023-09-29T06:14:18,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:18,038 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:14:18,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eefd39c60cf58e105fccf6c634940207:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:14:18,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:14:18,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eefd39c60cf58e105fccf6c634940207:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:14:18,039 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:14:18,040 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41978 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:14:18,040 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): eefd39c60cf58e105fccf6c634940207/A is initiating minor compaction (all files) 2023-09-29T06:14:18,040 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of eefd39c60cf58e105fccf6c634940207/A in TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:18,040 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/f4742bf9ffa14c61be6959892aad810a, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/add58ffd1a3348979c943c3c2aee8dce, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/f6ab128c2a5043efa34f082436768f34] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp, totalSize=41.0 K 2023-09-29T06:14:18,041 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37148 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:14:18,041 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): eefd39c60cf58e105fccf6c634940207/B is initiating minor compaction (all files) 2023-09-29T06:14:18,041 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of eefd39c60cf58e105fccf6c634940207/B in TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:18,041 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/4bc55da3d3ac43049556b842a22ef33c, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/93478b8d68cf4298b16dd9d82a666f44, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/b5a3035d5a08424b9319f18f231469ea] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp, totalSize=36.3 K 2023-09-29T06:14:18,041 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting f4742bf9ffa14c61be6959892aad810a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1695968055821 2023-09-29T06:14:18,041 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 4bc55da3d3ac43049556b842a22ef33c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1695968055821 2023-09-29T06:14:18,041 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting add58ffd1a3348979c943c3c2aee8dce, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1695968056452 2023-09-29T06:14:18,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:14:18,042 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 93478b8d68cf4298b16dd9d82a666f44, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1695968056456 2023-09-29T06:14:18,042 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting f6ab128c2a5043efa34f082436768f34, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1695968056591 2023-09-29T06:14:18,043 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting b5a3035d5a08424b9319f18f231469ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1695968056591 2023-09-29T06:14:18,051 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eefd39c60cf58e105fccf6c634940207#A#compaction#289 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:14:18,052 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/5f1611b55f4643aa8db0edae1eacec70 is 50, key is test_row_0/A:col10/1695968056709/Put/seqid=0 2023-09-29T06:14:18,053 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eefd39c60cf58e105fccf6c634940207#B#compaction#290 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:14:18,054 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/79b0f34119ed4d2f8ee1f593ca872521 is 50, key is test_row_0/B:col10/1695968056709/Put/seqid=0 2023-09-29T06:14:18,080 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/5f1611b55f4643aa8db0edae1eacec70 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/5f1611b55f4643aa8db0edae1eacec70 2023-09-29T06:14:18,085 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in eefd39c60cf58e105fccf6c634940207/A of eefd39c60cf58e105fccf6c634940207 into 5f1611b55f4643aa8db0edae1eacec70(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:14:18,085 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:18,085 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207., storeName=eefd39c60cf58e105fccf6c634940207/A, priority=13, startTime=1695968058038; duration=0sec 2023-09-29T06:14:18,085 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:14:18,085 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eefd39c60cf58e105fccf6c634940207:A 2023-09-29T06:14:18,085 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2023-09-29T06:14:18,085 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2023-09-29T06:14:18,085 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2023-09-29T06:14:18,085 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. because compaction request was cancelled 2023-09-29T06:14:18,086 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eefd39c60cf58e105fccf6c634940207:C 2023-09-29T06:14:18,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=80 2023-09-29T06:14:18,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:18,171 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HRegion(2819): Flushing eefd39c60cf58e105fccf6c634940207 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2023-09-29T06:14:18,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=A 2023-09-29T06:14:18,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:18,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=B 2023-09-29T06:14:18,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:18,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=C 2023-09-29T06:14:18,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:18,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/fca517171ae348179d165d92277badbc is 50, key is test_row_0/A:col10/1695968056719/Put/seqid=0 2023-09-29T06:14:18,192 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/fca517171ae348179d165d92277badbc 2023-09-29T06:14:18,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/9f675551499d44109d7d8e4db80d4a6a is 50, key is test_row_0/B:col10/1695968056719/Put/seqid=0 2023-09-29T06:14:18,245 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/9f675551499d44109d7d8e4db80d4a6a 2023-09-29T06:14:18,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/248aa285819245b3a2ad2fc69b8db9d1 is 50, key is test_row_0/C:col10/1695968056719/Put/seqid=0 2023-09-29T06:14:18,258 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/248aa285819245b3a2ad2fc69b8db9d1 2023-09-29T06:14:18,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/fca517171ae348179d165d92277badbc as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/fca517171ae348179d165d92277badbc 2023-09-29T06:14:18,275 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/fca517171ae348179d165d92277badbc, entries=150, sequenceid=298, filesize=12.0 K 2023-09-29T06:14:18,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/9f675551499d44109d7d8e4db80d4a6a as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/9f675551499d44109d7d8e4db80d4a6a 2023-09-29T06:14:18,283 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/9f675551499d44109d7d8e4db80d4a6a, entries=150, sequenceid=298, filesize=12.0 K 2023-09-29T06:14:18,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/248aa285819245b3a2ad2fc69b8db9d1 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/248aa285819245b3a2ad2fc69b8db9d1 2023-09-29T06:14:18,288 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/248aa285819245b3a2ad2fc69b8db9d1, entries=150, sequenceid=298, filesize=12.0 K 2023-09-29T06:14:18,288 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HRegion(3022): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=0 B/0 for eefd39c60cf58e105fccf6c634940207 in 117ms, sequenceid=298, compaction requested=true 2023-09-29T06:14:18,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HRegion(2520): Flush status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:18,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:18,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=80 2023-09-29T06:14:18,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=80 2023-09-29T06:14:18,293 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=80, resume processing ppid=79 2023-09-29T06:14:18,293 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=80, ppid=79, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0450 sec 2023-09-29T06:14:18,295 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=79, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=79, table=TestAcidGuarantees in 1.0570 sec 2023-09-29T06:14:18,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=79 2023-09-29T06:14:18,355 INFO [Thread-1523 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 79 completed 2023-09-29T06:14:18,356 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:14:18,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=81, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=81, table=TestAcidGuarantees 2023-09-29T06:14:18,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=81 2023-09-29T06:14:18,359 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=81, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=81, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:14:18,360 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=81, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=81, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:14:18,360 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:14:18,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=81 2023-09-29T06:14:18,486 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/79b0f34119ed4d2f8ee1f593ca872521 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/79b0f34119ed4d2f8ee1f593ca872521 2023-09-29T06:14:18,494 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in eefd39c60cf58e105fccf6c634940207/B of eefd39c60cf58e105fccf6c634940207 into 79b0f34119ed4d2f8ee1f593ca872521(size=12.6 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:14:18,494 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:18,494 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207., storeName=eefd39c60cf58e105fccf6c634940207/B, priority=13, startTime=1695968058038; duration=0sec 2023-09-29T06:14:18,494 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:18,494 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eefd39c60cf58e105fccf6c634940207:B 2023-09-29T06:14:18,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=82 2023-09-29T06:14:18,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:18,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegion(2520): Flush status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:18,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:18,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=82 2023-09-29T06:14:18,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=82 2023-09-29T06:14:18,518 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=82, resume processing ppid=81 2023-09-29T06:14:18,518 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=82, ppid=81, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 156 msec 2023-09-29T06:14:18,520 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=81, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=81, table=TestAcidGuarantees in 163 msec 2023-09-29T06:14:18,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=81 2023-09-29T06:14:18,661 INFO [Thread-1523 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 81 completed 2023-09-29T06:14:18,663 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:14:18,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=83, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=83, table=TestAcidGuarantees 2023-09-29T06:14:18,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=83 2023-09-29T06:14:18,665 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=83, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=83, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:14:18,666 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=83, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=83, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:14:18,667 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:14:18,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=83 2023-09-29T06:14:18,820 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=84 2023-09-29T06:14:18,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:18,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2520): Flush status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:18,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:18,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=84 2023-09-29T06:14:18,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=84 2023-09-29T06:14:18,826 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=84, resume processing ppid=83 2023-09-29T06:14:18,826 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=84, ppid=83, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 156 msec 2023-09-29T06:14:18,828 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=83, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=83, table=TestAcidGuarantees in 164 msec 2023-09-29T06:14:18,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on eefd39c60cf58e105fccf6c634940207 2023-09-29T06:14:18,852 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing eefd39c60cf58e105fccf6c634940207 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2023-09-29T06:14:18,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=A 2023-09-29T06:14:18,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:18,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=B 2023-09-29T06:14:18,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:18,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=C 2023-09-29T06:14:18,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:18,857 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/ab16378cf1754deb97bd6d4d2d094e0d is 50, key is test_row_0/A:col10/1695968058851/Put/seqid=0 2023-09-29T06:14:18,865 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/ab16378cf1754deb97bd6d4d2d094e0d 2023-09-29T06:14:18,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/1641b27f8ef54fd98b44f41140a30c78 is 50, key is test_row_0/B:col10/1695968058851/Put/seqid=0 2023-09-29T06:14:18,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:18,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968118875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:18,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:18,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968118878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:18,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:18,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968118880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:18,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:18,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968118879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:18,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=83 2023-09-29T06:14:18,967 INFO [Thread-1523 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 83 completed 2023-09-29T06:14:18,969 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:14:18,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=85, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees 2023-09-29T06:14:18,972 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=85, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:14:18,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=85 2023-09-29T06:14:18,973 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=85, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:14:18,974 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:14:18,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:18,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968118980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:18,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:18,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968118983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:18,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:18,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968118991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:18,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:18,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968118994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:19,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=85 2023-09-29T06:14:19,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2023-09-29T06:14:19,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:19,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. as already flushing 2023-09-29T06:14:19,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:19,129 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(53): pid=86 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:19,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:19,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:19,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:19,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968119185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:19,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:19,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968119188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:19,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:19,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968119195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:19,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:19,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968119198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:19,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=85 2023-09-29T06:14:19,281 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/1641b27f8ef54fd98b44f41140a30c78 2023-09-29T06:14:19,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2023-09-29T06:14:19,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:19,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. as already flushing 2023-09-29T06:14:19,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:19,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(53): pid=86 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:19,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:19,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:19,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/8e5955960bf9465eb12e6a5a28f4b520 is 50, key is test_row_0/C:col10/1695968058851/Put/seqid=0 2023-09-29T06:14:19,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2023-09-29T06:14:19,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:19,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. as already flushing 2023-09-29T06:14:19,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:19,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(53): pid=86 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:19,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:19,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:19,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:19,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968119489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:19,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:19,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968119493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:19,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:19,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968119498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:19,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:19,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968119501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:19,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=85 2023-09-29T06:14:19,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2023-09-29T06:14:19,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:19,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. as already flushing 2023-09-29T06:14:19,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:19,588 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(53): pid=86 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:19,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:19,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:19,694 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/8e5955960bf9465eb12e6a5a28f4b520 2023-09-29T06:14:19,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/ab16378cf1754deb97bd6d4d2d094e0d as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/ab16378cf1754deb97bd6d4d2d094e0d 2023-09-29T06:14:19,706 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/ab16378cf1754deb97bd6d4d2d094e0d, entries=150, sequenceid=314, filesize=12.0 K 2023-09-29T06:14:19,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/1641b27f8ef54fd98b44f41140a30c78 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/1641b27f8ef54fd98b44f41140a30c78 2023-09-29T06:14:19,715 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/1641b27f8ef54fd98b44f41140a30c78, entries=150, sequenceid=314, filesize=12.0 K 2023-09-29T06:14:19,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/8e5955960bf9465eb12e6a5a28f4b520 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/8e5955960bf9465eb12e6a5a28f4b520 2023-09-29T06:14:19,724 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/8e5955960bf9465eb12e6a5a28f4b520, entries=150, sequenceid=314, filesize=12.0 K 2023-09-29T06:14:19,724 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for eefd39c60cf58e105fccf6c634940207 in 872ms, sequenceid=314, compaction requested=true 2023-09-29T06:14:19,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:19,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eefd39c60cf58e105fccf6c634940207:A, priority=-2147483648, current under compaction store size is 1 2023-09-29T06:14:19,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:19,725 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:14:19,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eefd39c60cf58e105fccf6c634940207:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:14:19,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:19,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eefd39c60cf58e105fccf6c634940207:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:14:19,725 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:14:19,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:14:19,726 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37500 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:14:19,726 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): eefd39c60cf58e105fccf6c634940207/A is initiating minor compaction (all files) 2023-09-29T06:14:19,726 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37500 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:14:19,726 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of eefd39c60cf58e105fccf6c634940207/A in TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:19,726 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): eefd39c60cf58e105fccf6c634940207/B is initiating minor compaction (all files) 2023-09-29T06:14:19,727 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/5f1611b55f4643aa8db0edae1eacec70, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/fca517171ae348179d165d92277badbc, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/ab16378cf1754deb97bd6d4d2d094e0d] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp, totalSize=36.6 K 2023-09-29T06:14:19,727 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of eefd39c60cf58e105fccf6c634940207/B in TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:19,727 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/79b0f34119ed4d2f8ee1f593ca872521, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/9f675551499d44109d7d8e4db80d4a6a, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/1641b27f8ef54fd98b44f41140a30c78] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp, totalSize=36.6 K 2023-09-29T06:14:19,727 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 5f1611b55f4643aa8db0edae1eacec70, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1695968056591 2023-09-29T06:14:19,727 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting fca517171ae348179d165d92277badbc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1695968056712 2023-09-29T06:14:19,728 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 79b0f34119ed4d2f8ee1f593ca872521, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1695968056591 2023-09-29T06:14:19,728 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting ab16378cf1754deb97bd6d4d2d094e0d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1695968058849 2023-09-29T06:14:19,729 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 9f675551499d44109d7d8e4db80d4a6a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1695968056712 2023-09-29T06:14:19,729 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 1641b27f8ef54fd98b44f41140a30c78, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1695968058849 2023-09-29T06:14:19,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2023-09-29T06:14:19,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:19,741 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2819): Flushing eefd39c60cf58e105fccf6c634940207 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2023-09-29T06:14:19,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=A 2023-09-29T06:14:19,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:19,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=B 2023-09-29T06:14:19,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:19,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=C 2023-09-29T06:14:19,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:19,753 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eefd39c60cf58e105fccf6c634940207#A#compaction#297 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:14:19,754 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/d4917e280dbf4e4a8c0ed72524533b33 is 50, key is test_row_0/A:col10/1695968058851/Put/seqid=0 2023-09-29T06:14:19,755 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eefd39c60cf58e105fccf6c634940207#B#compaction#298 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:14:19,756 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/2c9cd09dd3a24b748c6dc960b7f015f6 is 50, key is test_row_0/B:col10/1695968058851/Put/seqid=0 2023-09-29T06:14:19,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/718a030fd1f043f28aa9f520d9d66521 is 50, key is test_row_0/A:col10/1695968058878/Put/seqid=0 2023-09-29T06:14:19,775 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/d4917e280dbf4e4a8c0ed72524533b33 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/d4917e280dbf4e4a8c0ed72524533b33 2023-09-29T06:14:19,780 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in eefd39c60cf58e105fccf6c634940207/A of eefd39c60cf58e105fccf6c634940207 into d4917e280dbf4e4a8c0ed72524533b33(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:14:19,780 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:19,780 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207., storeName=eefd39c60cf58e105fccf6c634940207/A, priority=13, startTime=1695968059725; duration=0sec 2023-09-29T06:14:19,780 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:14:19,780 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eefd39c60cf58e105fccf6c634940207:A 2023-09-29T06:14:19,780 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-29T06:14:19,781 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49616 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-29T06:14:19,781 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): eefd39c60cf58e105fccf6c634940207/C is initiating minor compaction (all files) 2023-09-29T06:14:19,782 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of eefd39c60cf58e105fccf6c634940207/C in TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:19,782 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/e4ddcef096f1475594dd4ce155f80fee, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/841e9b2f5e094b4c9f5dec742923aaeb, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/248aa285819245b3a2ad2fc69b8db9d1, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/8e5955960bf9465eb12e6a5a28f4b520] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp, totalSize=48.5 K 2023-09-29T06:14:19,786 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting e4ddcef096f1475594dd4ce155f80fee, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1695968056456 2023-09-29T06:14:19,786 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 841e9b2f5e094b4c9f5dec742923aaeb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1695968056591 2023-09-29T06:14:19,787 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 248aa285819245b3a2ad2fc69b8db9d1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1695968056712 2023-09-29T06:14:19,787 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 8e5955960bf9465eb12e6a5a28f4b520, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1695968058849 2023-09-29T06:14:19,805 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eefd39c60cf58e105fccf6c634940207#C#compaction#300 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:14:19,805 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/1189e7fc917c425b83efb7168df21020 is 50, key is test_row_0/C:col10/1695968058851/Put/seqid=0 2023-09-29T06:14:19,823 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/1189e7fc917c425b83efb7168df21020 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/1189e7fc917c425b83efb7168df21020 2023-09-29T06:14:19,828 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in eefd39c60cf58e105fccf6c634940207/C of eefd39c60cf58e105fccf6c634940207 into 1189e7fc917c425b83efb7168df21020(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:14:19,829 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:19,829 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207., storeName=eefd39c60cf58e105fccf6c634940207/C, priority=12, startTime=1695968059725; duration=0sec 2023-09-29T06:14:19,829 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:19,829 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eefd39c60cf58e105fccf6c634940207:C 2023-09-29T06:14:19,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on eefd39c60cf58e105fccf6c634940207 2023-09-29T06:14:19,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. as already flushing 2023-09-29T06:14:20,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:20,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968120039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:20,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:20,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968120040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:20,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:20,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968120041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:20,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:20,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968120042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:20,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=85 2023-09-29T06:14:20,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:20,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968120143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:20,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:20,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968120143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:20,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:20,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968120145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:20,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:20,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968120145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:20,205 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/718a030fd1f043f28aa9f520d9d66521 2023-09-29T06:14:20,215 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/2c9cd09dd3a24b748c6dc960b7f015f6 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/2c9cd09dd3a24b748c6dc960b7f015f6 2023-09-29T06:14:20,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/b49d38840372492986af86b75b629221 is 50, key is test_row_0/B:col10/1695968058878/Put/seqid=0 2023-09-29T06:14:20,225 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in eefd39c60cf58e105fccf6c634940207/B of eefd39c60cf58e105fccf6c634940207 into 2c9cd09dd3a24b748c6dc960b7f015f6(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:14:20,225 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:20,225 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207., storeName=eefd39c60cf58e105fccf6c634940207/B, priority=13, startTime=1695968059725; duration=0sec 2023-09-29T06:14:20,226 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:20,226 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eefd39c60cf58e105fccf6c634940207:B 2023-09-29T06:14:20,269 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/b49d38840372492986af86b75b629221 2023-09-29T06:14:20,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/bdaa8eea0fa8493d976d1e35dd398d1f is 50, key is test_row_0/C:col10/1695968058878/Put/seqid=0 2023-09-29T06:14:20,308 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/bdaa8eea0fa8493d976d1e35dd398d1f 2023-09-29T06:14:20,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/718a030fd1f043f28aa9f520d9d66521 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/718a030fd1f043f28aa9f520d9d66521 2023-09-29T06:14:20,320 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/718a030fd1f043f28aa9f520d9d66521, entries=150, sequenceid=337, filesize=12.0 K 2023-09-29T06:14:20,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/b49d38840372492986af86b75b629221 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/b49d38840372492986af86b75b629221 2023-09-29T06:14:20,327 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/b49d38840372492986af86b75b629221, entries=150, sequenceid=337, filesize=12.0 K 2023-09-29T06:14:20,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/bdaa8eea0fa8493d976d1e35dd398d1f as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/bdaa8eea0fa8493d976d1e35dd398d1f 2023-09-29T06:14:20,333 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/bdaa8eea0fa8493d976d1e35dd398d1f, entries=150, sequenceid=337, filesize=12.0 K 2023-09-29T06:14:20,334 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(3022): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for eefd39c60cf58e105fccf6c634940207 in 593ms, sequenceid=337, compaction requested=false 2023-09-29T06:14:20,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2520): Flush status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:20,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:20,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=86 2023-09-29T06:14:20,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=86 2023-09-29T06:14:20,339 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=86, resume processing ppid=85 2023-09-29T06:14:20,340 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=86, ppid=85, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3630 sec 2023-09-29T06:14:20,343 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=85, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees in 1.3720 sec 2023-09-29T06:14:20,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on eefd39c60cf58e105fccf6c634940207 2023-09-29T06:14:20,362 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing eefd39c60cf58e105fccf6c634940207 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2023-09-29T06:14:20,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=A 2023-09-29T06:14:20,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:20,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=B 2023-09-29T06:14:20,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:20,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=C 2023-09-29T06:14:20,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:20,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/725d96e5a1a04c5cad1b8b71b558c936 is 50, key is test_row_0/A:col10/1695968060362/Put/seqid=0 2023-09-29T06:14:20,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:20,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968120383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:20,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:20,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968120385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:20,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:20,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968120386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:20,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:20,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968120387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:20,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:20,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968120487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:20,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:20,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968120489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:20,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:20,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968120492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:20,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:20,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968120492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:20,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:20,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968120690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:20,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:20,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968120693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:20,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:20,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968120695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:20,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:20,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968120695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:20,811 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/725d96e5a1a04c5cad1b8b71b558c936 2023-09-29T06:14:20,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/097b1f03a9ae466fb6fc97ab588554be is 50, key is test_row_0/B:col10/1695968060362/Put/seqid=0 2023-09-29T06:14:20,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:20,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968120993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:20,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:20,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968120996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:21,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:21,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968120999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:21,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:21,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968120999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:21,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=85 2023-09-29T06:14:21,077 INFO [Thread-1523 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 85 completed 2023-09-29T06:14:21,078 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:14:21,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees 2023-09-29T06:14:21,082 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:14:21,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=87 2023-09-29T06:14:21,082 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=87, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:14:21,082 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:14:21,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=87 2023-09-29T06:14:21,236 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/097b1f03a9ae466fb6fc97ab588554be 2023-09-29T06:14:21,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2023-09-29T06:14:21,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:21,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. as already flushing 2023-09-29T06:14:21,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:21,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(53): pid=88 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:21,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:21,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:21,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/d47be896ae06447aa77ba20810b9dac2 is 50, key is test_row_0/C:col10/1695968060362/Put/seqid=0 2023-09-29T06:14:21,259 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/d47be896ae06447aa77ba20810b9dac2 2023-09-29T06:14:21,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/725d96e5a1a04c5cad1b8b71b558c936 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/725d96e5a1a04c5cad1b8b71b558c936 2023-09-29T06:14:21,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/725d96e5a1a04c5cad1b8b71b558c936, entries=150, sequenceid=357, filesize=12.0 K 2023-09-29T06:14:21,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/097b1f03a9ae466fb6fc97ab588554be as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/097b1f03a9ae466fb6fc97ab588554be 2023-09-29T06:14:21,272 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/097b1f03a9ae466fb6fc97ab588554be, entries=150, sequenceid=357, filesize=12.0 K 2023-09-29T06:14:21,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/d47be896ae06447aa77ba20810b9dac2 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/d47be896ae06447aa77ba20810b9dac2 2023-09-29T06:14:21,283 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/d47be896ae06447aa77ba20810b9dac2, entries=150, sequenceid=357, filesize=12.0 K 2023-09-29T06:14:21,286 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for eefd39c60cf58e105fccf6c634940207 in 924ms, sequenceid=357, compaction requested=true 2023-09-29T06:14:21,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:21,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eefd39c60cf58e105fccf6c634940207:A, priority=-2147483648, current under compaction store size is 1 2023-09-29T06:14:21,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:21,286 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:14:21,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eefd39c60cf58e105fccf6c634940207:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:14:21,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:21,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eefd39c60cf58e105fccf6c634940207:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:14:21,286 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:14:21,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:14:21,288 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37602 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:14:21,288 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): eefd39c60cf58e105fccf6c634940207/A is initiating minor compaction (all files) 2023-09-29T06:14:21,288 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of eefd39c60cf58e105fccf6c634940207/A in TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:21,288 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/d4917e280dbf4e4a8c0ed72524533b33, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/718a030fd1f043f28aa9f520d9d66521, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/725d96e5a1a04c5cad1b8b71b558c936] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp, totalSize=36.7 K 2023-09-29T06:14:21,289 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37602 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:14:21,289 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting d4917e280dbf4e4a8c0ed72524533b33, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1695968058849 2023-09-29T06:14:21,289 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): eefd39c60cf58e105fccf6c634940207/B is initiating minor compaction (all files) 2023-09-29T06:14:21,289 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of eefd39c60cf58e105fccf6c634940207/B in TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:21,289 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/2c9cd09dd3a24b748c6dc960b7f015f6, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/b49d38840372492986af86b75b629221, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/097b1f03a9ae466fb6fc97ab588554be] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp, totalSize=36.7 K 2023-09-29T06:14:21,290 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 718a030fd1f043f28aa9f520d9d66521, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1695968058874 2023-09-29T06:14:21,290 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 2c9cd09dd3a24b748c6dc960b7f015f6, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1695968058849 2023-09-29T06:14:21,290 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 725d96e5a1a04c5cad1b8b71b558c936, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1695968060040 2023-09-29T06:14:21,291 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting b49d38840372492986af86b75b629221, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1695968058874 2023-09-29T06:14:21,291 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 097b1f03a9ae466fb6fc97ab588554be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1695968060040 2023-09-29T06:14:21,299 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eefd39c60cf58e105fccf6c634940207#A#compaction#306 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:14:21,299 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/4b0573c5b22d478398bf534c441301c3 is 50, key is test_row_0/A:col10/1695968060362/Put/seqid=0 2023-09-29T06:14:21,307 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eefd39c60cf58e105fccf6c634940207#B#compaction#307 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:14:21,307 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/f138c17e87cc4b1dad17ec9ba29cfeb1 is 50, key is test_row_0/B:col10/1695968060362/Put/seqid=0 2023-09-29T06:14:21,332 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/4b0573c5b22d478398bf534c441301c3 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/4b0573c5b22d478398bf534c441301c3 2023-09-29T06:14:21,336 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in eefd39c60cf58e105fccf6c634940207/A of eefd39c60cf58e105fccf6c634940207 into 4b0573c5b22d478398bf534c441301c3(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:14:21,337 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:21,337 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207., storeName=eefd39c60cf58e105fccf6c634940207/A, priority=13, startTime=1695968061286; duration=0sec 2023-09-29T06:14:21,337 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:14:21,337 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eefd39c60cf58e105fccf6c634940207:A 2023-09-29T06:14:21,337 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:14:21,338 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37568 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:14:21,338 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): eefd39c60cf58e105fccf6c634940207/C is initiating minor compaction (all files) 2023-09-29T06:14:21,338 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of eefd39c60cf58e105fccf6c634940207/C in TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:21,338 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/1189e7fc917c425b83efb7168df21020, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/bdaa8eea0fa8493d976d1e35dd398d1f, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/d47be896ae06447aa77ba20810b9dac2] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp, totalSize=36.7 K 2023-09-29T06:14:21,338 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 1189e7fc917c425b83efb7168df21020, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1695968058849 2023-09-29T06:14:21,339 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting bdaa8eea0fa8493d976d1e35dd398d1f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1695968058874 2023-09-29T06:14:21,339 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting d47be896ae06447aa77ba20810b9dac2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1695968060040 2023-09-29T06:14:21,352 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eefd39c60cf58e105fccf6c634940207#C#compaction#308 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:14:21,353 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/4ab930775417401ca23b2c50902eb32c is 50, key is test_row_0/C:col10/1695968060362/Put/seqid=0 2023-09-29T06:14:21,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=87 2023-09-29T06:14:21,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2023-09-29T06:14:21,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:21,393 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2819): Flushing eefd39c60cf58e105fccf6c634940207 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2023-09-29T06:14:21,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=A 2023-09-29T06:14:21,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:21,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=B 2023-09-29T06:14:21,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:21,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=C 2023-09-29T06:14:21,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:21,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/fd7a5ec6038e486ba715bacfd17fc5bd is 50, key is test_row_0/A:col10/1695968060384/Put/seqid=0 2023-09-29T06:14:21,404 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/fd7a5ec6038e486ba715bacfd17fc5bd 2023-09-29T06:14:21,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/9e98700d1060442daa5f885643022801 is 50, key is test_row_0/B:col10/1695968060384/Put/seqid=0 2023-09-29T06:14:21,442 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/9e98700d1060442daa5f885643022801 2023-09-29T06:14:21,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/7606469a95884e0aa4563a5b61029eb7 is 50, key is test_row_0/C:col10/1695968060384/Put/seqid=0 2023-09-29T06:14:21,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on eefd39c60cf58e105fccf6c634940207 2023-09-29T06:14:21,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. as already flushing 2023-09-29T06:14:21,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:21,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968121512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:21,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:21,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968121513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:21,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:21,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968121514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:21,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:21,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968121516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:21,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:21,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968121617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:21,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:21,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968121617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:21,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:21,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968121618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:21,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:21,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968121618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:21,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=87 2023-09-29T06:14:21,735 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/f138c17e87cc4b1dad17ec9ba29cfeb1 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/f138c17e87cc4b1dad17ec9ba29cfeb1 2023-09-29T06:14:21,740 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in eefd39c60cf58e105fccf6c634940207/B of eefd39c60cf58e105fccf6c634940207 into f138c17e87cc4b1dad17ec9ba29cfeb1(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:14:21,740 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:21,740 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207., storeName=eefd39c60cf58e105fccf6c634940207/B, priority=13, startTime=1695968061286; duration=0sec 2023-09-29T06:14:21,740 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:21,740 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eefd39c60cf58e105fccf6c634940207:B 2023-09-29T06:14:21,776 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/4ab930775417401ca23b2c50902eb32c as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/4ab930775417401ca23b2c50902eb32c 2023-09-29T06:14:21,783 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in eefd39c60cf58e105fccf6c634940207/C of eefd39c60cf58e105fccf6c634940207 into 4ab930775417401ca23b2c50902eb32c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:14:21,783 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:21,783 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207., storeName=eefd39c60cf58e105fccf6c634940207/C, priority=13, startTime=1695968061286; duration=0sec 2023-09-29T06:14:21,783 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:21,783 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eefd39c60cf58e105fccf6c634940207:C 2023-09-29T06:14:21,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:21,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968121821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:21,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:21,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968121821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:21,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:21,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968121821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:21,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:21,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968121821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:21,870 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/7606469a95884e0aa4563a5b61029eb7 2023-09-29T06:14:21,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/fd7a5ec6038e486ba715bacfd17fc5bd as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/fd7a5ec6038e486ba715bacfd17fc5bd 2023-09-29T06:14:21,891 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/fd7a5ec6038e486ba715bacfd17fc5bd, entries=150, sequenceid=378, filesize=12.0 K 2023-09-29T06:14:21,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/9e98700d1060442daa5f885643022801 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/9e98700d1060442daa5f885643022801 2023-09-29T06:14:21,897 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/9e98700d1060442daa5f885643022801, entries=150, sequenceid=378, filesize=12.0 K 2023-09-29T06:14:21,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/7606469a95884e0aa4563a5b61029eb7 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/7606469a95884e0aa4563a5b61029eb7 2023-09-29T06:14:21,903 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/7606469a95884e0aa4563a5b61029eb7, entries=150, sequenceid=378, filesize=12.0 K 2023-09-29T06:14:21,904 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(3022): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for eefd39c60cf58e105fccf6c634940207 in 511ms, sequenceid=378, compaction requested=false 2023-09-29T06:14:21,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2520): Flush status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:21,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:21,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=88 2023-09-29T06:14:21,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=88 2023-09-29T06:14:21,911 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=88, resume processing ppid=87 2023-09-29T06:14:21,911 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=88, ppid=87, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 823 msec 2023-09-29T06:14:21,912 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=87, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees in 833 msec 2023-09-29T06:14:22,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on eefd39c60cf58e105fccf6c634940207 2023-09-29T06:14:22,128 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing eefd39c60cf58e105fccf6c634940207 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2023-09-29T06:14:22,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=A 2023-09-29T06:14:22,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:22,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=B 2023-09-29T06:14:22,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:22,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=C 2023-09-29T06:14:22,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:22,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/8a02a89774714f71b7ce1dd3c10034dc is 50, key is test_row_0/A:col10/1695968062128/Put/seqid=0 2023-09-29T06:14:22,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:22,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968122144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:22,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:22,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968122145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:22,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:22,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:22,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968122145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:22,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968122146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:22,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=87 2023-09-29T06:14:22,187 INFO [Thread-1523 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 87 completed 2023-09-29T06:14:22,188 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:14:22,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees 2023-09-29T06:14:22,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=89 2023-09-29T06:14:22,196 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:14:22,196 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=89, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:14:22,197 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:14:22,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:22,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968122248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:22,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:22,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968122249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:22,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:22,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968122250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:22,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:22,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968122250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:22,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=89 2023-09-29T06:14:22,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2023-09-29T06:14:22,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:22,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. as already flushing 2023-09-29T06:14:22,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:22,351 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(53): pid=90 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:22,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:22,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:22,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:22,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968122450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:22,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:22,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968122452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:22,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:22,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968122452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:22,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:22,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968122453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:22,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=89 2023-09-29T06:14:22,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2023-09-29T06:14:22,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:22,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. as already flushing 2023-09-29T06:14:22,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:22,504 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(53): pid=90 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:22,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:22,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:22,552 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/8a02a89774714f71b7ce1dd3c10034dc 2023-09-29T06:14:22,560 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/cfe0680b839a4c69b29ba596bf204113 is 50, key is test_row_0/B:col10/1695968062128/Put/seqid=0 2023-09-29T06:14:22,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2023-09-29T06:14:22,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:22,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. as already flushing 2023-09-29T06:14:22,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:22,657 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(53): pid=90 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:22,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:22,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:22,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:22,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968122753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:22,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:22,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968122756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:22,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:22,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968122757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:22,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:22,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968122758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:22,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=89 2023-09-29T06:14:22,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2023-09-29T06:14:22,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:22,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. as already flushing 2023-09-29T06:14:22,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:22,810 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(53): pid=90 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:22,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:22,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:22,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2023-09-29T06:14:22,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:22,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. as already flushing 2023-09-29T06:14:22,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:22,963 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(53): pid=90 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:22,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:22,973 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/cfe0680b839a4c69b29ba596bf204113 2023-09-29T06:14:22,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/d3054ed7a8a44c3f826e32a5dbed42ac is 50, key is test_row_0/C:col10/1695968062128/Put/seqid=0 2023-09-29T06:14:23,006 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/d3054ed7a8a44c3f826e32a5dbed42ac 2023-09-29T06:14:23,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/8a02a89774714f71b7ce1dd3c10034dc as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/8a02a89774714f71b7ce1dd3c10034dc 2023-09-29T06:14:23,014 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/8a02a89774714f71b7ce1dd3c10034dc, entries=150, sequenceid=399, filesize=12.0 K 2023-09-29T06:14:23,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/cfe0680b839a4c69b29ba596bf204113 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/cfe0680b839a4c69b29ba596bf204113 2023-09-29T06:14:23,019 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/cfe0680b839a4c69b29ba596bf204113, entries=150, sequenceid=399, filesize=12.0 K 2023-09-29T06:14:23,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/d3054ed7a8a44c3f826e32a5dbed42ac as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/d3054ed7a8a44c3f826e32a5dbed42ac 2023-09-29T06:14:23,023 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/d3054ed7a8a44c3f826e32a5dbed42ac, entries=150, sequenceid=399, filesize=12.0 K 2023-09-29T06:14:23,024 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for eefd39c60cf58e105fccf6c634940207 in 896ms, sequenceid=399, compaction requested=true 2023-09-29T06:14:23,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:23,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eefd39c60cf58e105fccf6c634940207:A, priority=-2147483648, current under compaction store size is 1 2023-09-29T06:14:23,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:23,024 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:14:23,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eefd39c60cf58e105fccf6c634940207:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:14:23,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:23,024 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:14:23,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eefd39c60cf58e105fccf6c634940207:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:14:23,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:14:23,026 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37704 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:14:23,026 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): eefd39c60cf58e105fccf6c634940207/A is initiating minor compaction (all files) 2023-09-29T06:14:23,026 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of eefd39c60cf58e105fccf6c634940207/A in TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:23,026 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/4b0573c5b22d478398bf534c441301c3, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/fd7a5ec6038e486ba715bacfd17fc5bd, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/8a02a89774714f71b7ce1dd3c10034dc] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp, totalSize=36.8 K 2023-09-29T06:14:23,026 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37704 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:14:23,026 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): eefd39c60cf58e105fccf6c634940207/B is initiating minor compaction (all files) 2023-09-29T06:14:23,026 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of eefd39c60cf58e105fccf6c634940207/B in TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:23,026 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/f138c17e87cc4b1dad17ec9ba29cfeb1, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/9e98700d1060442daa5f885643022801, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/cfe0680b839a4c69b29ba596bf204113] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp, totalSize=36.8 K 2023-09-29T06:14:23,026 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 4b0573c5b22d478398bf534c441301c3, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1695968060040 2023-09-29T06:14:23,026 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting f138c17e87cc4b1dad17ec9ba29cfeb1, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1695968060040 2023-09-29T06:14:23,027 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting fd7a5ec6038e486ba715bacfd17fc5bd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1695968060384 2023-09-29T06:14:23,027 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 9e98700d1060442daa5f885643022801, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1695968060384 2023-09-29T06:14:23,027 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 8a02a89774714f71b7ce1dd3c10034dc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1695968061511 2023-09-29T06:14:23,028 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting cfe0680b839a4c69b29ba596bf204113, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1695968061511 2023-09-29T06:14:23,034 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eefd39c60cf58e105fccf6c634940207#B#compaction#315 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:14:23,034 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/ec1b9d53de6c458fa878c195384ea51a is 50, key is test_row_0/B:col10/1695968062128/Put/seqid=0 2023-09-29T06:14:23,044 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eefd39c60cf58e105fccf6c634940207#A#compaction#316 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:14:23,045 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/0d2272e59f0b4ada8eeaa90d1c1a90ce is 50, key is test_row_0/A:col10/1695968062128/Put/seqid=0 2023-09-29T06:14:23,059 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/ec1b9d53de6c458fa878c195384ea51a as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/ec1b9d53de6c458fa878c195384ea51a 2023-09-29T06:14:23,064 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in eefd39c60cf58e105fccf6c634940207/B of eefd39c60cf58e105fccf6c634940207 into ec1b9d53de6c458fa878c195384ea51a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:14:23,064 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:23,064 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207., storeName=eefd39c60cf58e105fccf6c634940207/B, priority=13, startTime=1695968063024; duration=0sec 2023-09-29T06:14:23,064 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:14:23,064 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eefd39c60cf58e105fccf6c634940207:B 2023-09-29T06:14:23,064 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:14:23,065 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37670 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:14:23,065 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): eefd39c60cf58e105fccf6c634940207/C is initiating minor compaction (all files) 2023-09-29T06:14:23,065 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of eefd39c60cf58e105fccf6c634940207/C in TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:23,065 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/4ab930775417401ca23b2c50902eb32c, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/7606469a95884e0aa4563a5b61029eb7, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/d3054ed7a8a44c3f826e32a5dbed42ac] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp, totalSize=36.8 K 2023-09-29T06:14:23,066 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 4ab930775417401ca23b2c50902eb32c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1695968060040 2023-09-29T06:14:23,067 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 7606469a95884e0aa4563a5b61029eb7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1695968060384 2023-09-29T06:14:23,070 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting d3054ed7a8a44c3f826e32a5dbed42ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1695968061511 2023-09-29T06:14:23,075 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/0d2272e59f0b4ada8eeaa90d1c1a90ce as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/0d2272e59f0b4ada8eeaa90d1c1a90ce 2023-09-29T06:14:23,080 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in eefd39c60cf58e105fccf6c634940207/A of eefd39c60cf58e105fccf6c634940207 into 0d2272e59f0b4ada8eeaa90d1c1a90ce(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:14:23,080 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:23,080 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207., storeName=eefd39c60cf58e105fccf6c634940207/A, priority=13, startTime=1695968063024; duration=0sec 2023-09-29T06:14:23,080 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:23,080 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eefd39c60cf58e105fccf6c634940207:A 2023-09-29T06:14:23,083 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eefd39c60cf58e105fccf6c634940207#C#compaction#317 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:14:23,084 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/f4aef31651344ae6905da169d4d9d459 is 50, key is test_row_0/C:col10/1695968062128/Put/seqid=0 2023-09-29T06:14:23,108 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/f4aef31651344ae6905da169d4d9d459 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/f4aef31651344ae6905da169d4d9d459 2023-09-29T06:14:23,113 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in eefd39c60cf58e105fccf6c634940207/C of eefd39c60cf58e105fccf6c634940207 into f4aef31651344ae6905da169d4d9d459(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:14:23,113 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:23,113 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207., storeName=eefd39c60cf58e105fccf6c634940207/C, priority=13, startTime=1695968063024; duration=0sec 2023-09-29T06:14:23,113 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:23,113 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eefd39c60cf58e105fccf6c634940207:C 2023-09-29T06:14:23,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2023-09-29T06:14:23,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:23,116 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2819): Flushing eefd39c60cf58e105fccf6c634940207 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2023-09-29T06:14:23,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=A 2023-09-29T06:14:23,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:23,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=B 2023-09-29T06:14:23,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:23,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=C 2023-09-29T06:14:23,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:23,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/7ad6aab2f06d438c99ffe5623e882bcd is 50, key is test_row_0/A:col10/1695968062137/Put/seqid=0 2023-09-29T06:14:23,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on eefd39c60cf58e105fccf6c634940207 2023-09-29T06:14:23,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. as already flushing 2023-09-29T06:14:23,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:23,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968123273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:23,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:23,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968123275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:23,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:23,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:23,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968123277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:23,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968123278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:23,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=89 2023-09-29T06:14:23,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:23,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:23,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968123378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:23,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968123379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:23,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:23,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968123380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:23,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:23,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968123381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:23,576 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/7ad6aab2f06d438c99ffe5623e882bcd 2023-09-29T06:14:23,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:23,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968123582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:23,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:23,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968123582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:23,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:23,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968123584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:23,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:23,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968123585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:23,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/8f4009e2b9ae4700bfe50a650be78bd5 is 50, key is test_row_0/B:col10/1695968062137/Put/seqid=0 2023-09-29T06:14:23,604 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/8f4009e2b9ae4700bfe50a650be78bd5 2023-09-29T06:14:23,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/065c5607fb2146ddb2148dba6dfe8cf5 is 50, key is test_row_0/C:col10/1695968062137/Put/seqid=0 2023-09-29T06:14:23,629 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/065c5607fb2146ddb2148dba6dfe8cf5 2023-09-29T06:14:23,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/7ad6aab2f06d438c99ffe5623e882bcd as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/7ad6aab2f06d438c99ffe5623e882bcd 2023-09-29T06:14:23,639 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/7ad6aab2f06d438c99ffe5623e882bcd, entries=150, sequenceid=419, filesize=12.0 K 2023-09-29T06:14:23,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/8f4009e2b9ae4700bfe50a650be78bd5 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/8f4009e2b9ae4700bfe50a650be78bd5 2023-09-29T06:14:23,650 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/8f4009e2b9ae4700bfe50a650be78bd5, entries=150, sequenceid=419, filesize=12.0 K 2023-09-29T06:14:23,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/065c5607fb2146ddb2148dba6dfe8cf5 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/065c5607fb2146ddb2148dba6dfe8cf5 2023-09-29T06:14:23,664 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/065c5607fb2146ddb2148dba6dfe8cf5, entries=150, sequenceid=419, filesize=12.0 K 2023-09-29T06:14:23,664 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(3022): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for eefd39c60cf58e105fccf6c634940207 in 549ms, sequenceid=419, compaction requested=false 2023-09-29T06:14:23,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2520): Flush status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:23,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:23,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=90 2023-09-29T06:14:23,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=90 2023-09-29T06:14:23,667 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=90, resume processing ppid=89 2023-09-29T06:14:23,667 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=90, ppid=89, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4680 sec 2023-09-29T06:14:23,679 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=89, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees in 1.4820 sec 2023-09-29T06:14:23,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on eefd39c60cf58e105fccf6c634940207 2023-09-29T06:14:23,888 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing eefd39c60cf58e105fccf6c634940207 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2023-09-29T06:14:23,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=A 2023-09-29T06:14:23,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:23,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=B 2023-09-29T06:14:23,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:23,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=C 2023-09-29T06:14:23,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:23,893 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/9ce964a4295c4b999ac1452397c93d2f is 50, key is test_row_0/A:col10/1695968063275/Put/seqid=0 2023-09-29T06:14:23,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:23,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968123901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:23,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:23,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968123902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:23,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:23,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968123911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:23,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:23,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968123915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:24,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:24,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968124007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:24,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:24,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968124007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:24,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:24,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968124015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:24,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:24,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968124018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:24,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:24,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968124210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:24,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:24,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968124211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:24,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:24,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968124218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:24,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:24,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968124220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:24,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=89 2023-09-29T06:14:24,300 INFO [Thread-1523 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 89 completed 2023-09-29T06:14:24,300 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/9ce964a4295c4b999ac1452397c93d2f 2023-09-29T06:14:24,302 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:14:24,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2023-09-29T06:14:24,304 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:14:24,305 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:14:24,305 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:14:24,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=91 2023-09-29T06:14:24,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/a299b104283140e3a42df685b74a2928 is 50, key is test_row_0/B:col10/1695968063275/Put/seqid=0 2023-09-29T06:14:24,349 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/a299b104283140e3a42df685b74a2928 2023-09-29T06:14:24,369 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/083688467a384d128b298ad364a697b4 is 50, key is test_row_0/C:col10/1695968063275/Put/seqid=0 2023-09-29T06:14:24,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=91 2023-09-29T06:14:24,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2023-09-29T06:14:24,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:24,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. as already flushing 2023-09-29T06:14:24,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:24,463 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(53): pid=92 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:24,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:24,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:24,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:24,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968124512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:24,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:24,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968124514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:24,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:24,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968124521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:24,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:24,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968124524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:24,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=91 2023-09-29T06:14:24,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2023-09-29T06:14:24,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:24,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. as already flushing 2023-09-29T06:14:24,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:24,616 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(53): pid=92 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:24,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:24,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:24,769 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2023-09-29T06:14:24,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:24,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. as already flushing 2023-09-29T06:14:24,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:24,769 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(53): pid=92 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:24,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:24,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:24,785 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/083688467a384d128b298ad364a697b4 2023-09-29T06:14:24,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/9ce964a4295c4b999ac1452397c93d2f as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/9ce964a4295c4b999ac1452397c93d2f 2023-09-29T06:14:24,821 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/9ce964a4295c4b999ac1452397c93d2f, entries=150, sequenceid=440, filesize=12.0 K 2023-09-29T06:14:24,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/a299b104283140e3a42df685b74a2928 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/a299b104283140e3a42df685b74a2928 2023-09-29T06:14:24,832 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/a299b104283140e3a42df685b74a2928, entries=150, sequenceid=440, filesize=12.0 K 2023-09-29T06:14:24,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/083688467a384d128b298ad364a697b4 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/083688467a384d128b298ad364a697b4 2023-09-29T06:14:24,855 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/083688467a384d128b298ad364a697b4, entries=150, sequenceid=440, filesize=12.0 K 2023-09-29T06:14:24,856 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for eefd39c60cf58e105fccf6c634940207 in 968ms, sequenceid=440, compaction requested=true 2023-09-29T06:14:24,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:24,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eefd39c60cf58e105fccf6c634940207:A, priority=-2147483648, current under compaction store size is 1 2023-09-29T06:14:24,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:24,857 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:14:24,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eefd39c60cf58e105fccf6c634940207:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:14:24,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:24,857 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:14:24,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eefd39c60cf58e105fccf6c634940207:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:14:24,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:14:24,859 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37806 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:14:24,859 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): eefd39c60cf58e105fccf6c634940207/B is initiating minor compaction (all files) 2023-09-29T06:14:24,859 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of eefd39c60cf58e105fccf6c634940207/B in TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:24,859 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/ec1b9d53de6c458fa878c195384ea51a, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/8f4009e2b9ae4700bfe50a650be78bd5, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/a299b104283140e3a42df685b74a2928] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp, totalSize=36.9 K 2023-09-29T06:14:24,859 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37806 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:14:24,859 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): eefd39c60cf58e105fccf6c634940207/A is initiating minor compaction (all files) 2023-09-29T06:14:24,859 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of eefd39c60cf58e105fccf6c634940207/A in TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:24,859 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/0d2272e59f0b4ada8eeaa90d1c1a90ce, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/7ad6aab2f06d438c99ffe5623e882bcd, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/9ce964a4295c4b999ac1452397c93d2f] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp, totalSize=36.9 K 2023-09-29T06:14:24,860 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting ec1b9d53de6c458fa878c195384ea51a, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1695968061511 2023-09-29T06:14:24,860 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 0d2272e59f0b4ada8eeaa90d1c1a90ce, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1695968061511 2023-09-29T06:14:24,860 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 8f4009e2b9ae4700bfe50a650be78bd5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1695968062137 2023-09-29T06:14:24,860 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 7ad6aab2f06d438c99ffe5623e882bcd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1695968062137 2023-09-29T06:14:24,861 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting a299b104283140e3a42df685b74a2928, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1695968063275 2023-09-29T06:14:24,861 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 9ce964a4295c4b999ac1452397c93d2f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1695968063275 2023-09-29T06:14:24,880 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eefd39c60cf58e105fccf6c634940207#B#compaction#324 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:14:24,880 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/761d5954d6154b18aced7e6c37f280aa is 50, key is test_row_0/B:col10/1695968063275/Put/seqid=0 2023-09-29T06:14:24,901 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eefd39c60cf58e105fccf6c634940207#A#compaction#325 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:14:24,902 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/6b09fd12c1714ea0ac2ea2c664d23588 is 50, key is test_row_0/A:col10/1695968063275/Put/seqid=0 2023-09-29T06:14:24,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=91 2023-09-29T06:14:24,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2023-09-29T06:14:24,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:24,922 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2819): Flushing eefd39c60cf58e105fccf6c634940207 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2023-09-29T06:14:24,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=A 2023-09-29T06:14:24,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:24,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=B 2023-09-29T06:14:24,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:24,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=C 2023-09-29T06:14:24,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:24,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/ae6e5eb744e943b58ec086aa8af5f4e5 is 50, key is test_row_0/A:col10/1695968063904/Put/seqid=0 2023-09-29T06:14:24,943 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/761d5954d6154b18aced7e6c37f280aa as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/761d5954d6154b18aced7e6c37f280aa 2023-09-29T06:14:24,952 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in eefd39c60cf58e105fccf6c634940207/B of eefd39c60cf58e105fccf6c634940207 into 761d5954d6154b18aced7e6c37f280aa(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:14:24,953 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:24,953 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207., storeName=eefd39c60cf58e105fccf6c634940207/B, priority=13, startTime=1695968064857; duration=0sec 2023-09-29T06:14:24,953 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:14:24,953 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eefd39c60cf58e105fccf6c634940207:B 2023-09-29T06:14:24,953 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:14:24,954 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37772 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:14:24,955 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): eefd39c60cf58e105fccf6c634940207/C is initiating minor compaction (all files) 2023-09-29T06:14:24,955 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of eefd39c60cf58e105fccf6c634940207/C in TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:24,955 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/f4aef31651344ae6905da169d4d9d459, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/065c5607fb2146ddb2148dba6dfe8cf5, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/083688467a384d128b298ad364a697b4] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp, totalSize=36.9 K 2023-09-29T06:14:24,955 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting f4aef31651344ae6905da169d4d9d459, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1695968061511 2023-09-29T06:14:24,955 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 065c5607fb2146ddb2148dba6dfe8cf5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1695968062137 2023-09-29T06:14:24,956 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 083688467a384d128b298ad364a697b4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1695968063275 2023-09-29T06:14:24,959 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/ae6e5eb744e943b58ec086aa8af5f4e5 2023-09-29T06:14:24,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/8eff75d6da144372b0afe79378fbfed0 is 50, key is test_row_0/B:col10/1695968063904/Put/seqid=0 2023-09-29T06:14:24,967 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eefd39c60cf58e105fccf6c634940207#C#compaction#328 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:14:24,968 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/7c1fdce5038a4ecfaf79e455eced7487 is 50, key is test_row_0/C:col10/1695968063275/Put/seqid=0 2023-09-29T06:14:25,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on eefd39c60cf58e105fccf6c634940207 2023-09-29T06:14:25,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. as already flushing 2023-09-29T06:14:25,034 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/7c1fdce5038a4ecfaf79e455eced7487 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/7c1fdce5038a4ecfaf79e455eced7487 2023-09-29T06:14:25,039 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in eefd39c60cf58e105fccf6c634940207/C of eefd39c60cf58e105fccf6c634940207 into 7c1fdce5038a4ecfaf79e455eced7487(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:14:25,039 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:25,039 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207., storeName=eefd39c60cf58e105fccf6c634940207/C, priority=13, startTime=1695968064857; duration=0sec 2023-09-29T06:14:25,039 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:25,039 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eefd39c60cf58e105fccf6c634940207:C 2023-09-29T06:14:25,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:25,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968125045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:25,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:25,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968125047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:25,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:25,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968125049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:25,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:25,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968125049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:25,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:25,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968125149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:25,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:25,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968125150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:25,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:25,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968125152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:25,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:25,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968125152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:25,342 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/6b09fd12c1714ea0ac2ea2c664d23588 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/6b09fd12c1714ea0ac2ea2c664d23588 2023-09-29T06:14:25,346 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in eefd39c60cf58e105fccf6c634940207/A of eefd39c60cf58e105fccf6c634940207 into 6b09fd12c1714ea0ac2ea2c664d23588(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:14:25,346 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:25,346 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207., storeName=eefd39c60cf58e105fccf6c634940207/A, priority=13, startTime=1695968064857; duration=0sec 2023-09-29T06:14:25,346 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:25,346 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eefd39c60cf58e105fccf6c634940207:A 2023-09-29T06:14:25,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:25,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968125353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:25,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:25,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968125355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:25,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:25,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968125355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:25,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:25,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968125355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:25,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=91 2023-09-29T06:14:25,416 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/8eff75d6da144372b0afe79378fbfed0 2023-09-29T06:14:25,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/c0eefdd6d1e440c6aad214de68895a42 is 50, key is test_row_0/C:col10/1695968063904/Put/seqid=0 2023-09-29T06:14:25,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:25,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968125658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:25,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:25,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968125658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:25,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:25,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968125659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:25,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:25,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968125666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:25,844 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/c0eefdd6d1e440c6aad214de68895a42 2023-09-29T06:14:25,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/ae6e5eb744e943b58ec086aa8af5f4e5 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/ae6e5eb744e943b58ec086aa8af5f4e5 2023-09-29T06:14:25,853 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/ae6e5eb744e943b58ec086aa8af5f4e5, entries=150, sequenceid=456, filesize=12.0 K 2023-09-29T06:14:25,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/8eff75d6da144372b0afe79378fbfed0 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/8eff75d6da144372b0afe79378fbfed0 2023-09-29T06:14:25,861 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/8eff75d6da144372b0afe79378fbfed0, entries=150, sequenceid=456, filesize=12.0 K 2023-09-29T06:14:25,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/c0eefdd6d1e440c6aad214de68895a42 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/c0eefdd6d1e440c6aad214de68895a42 2023-09-29T06:14:25,883 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/c0eefdd6d1e440c6aad214de68895a42, entries=150, sequenceid=456, filesize=12.0 K 2023-09-29T06:14:25,884 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(3022): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for eefd39c60cf58e105fccf6c634940207 in 962ms, sequenceid=456, compaction requested=false 2023-09-29T06:14:25,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2520): Flush status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:25,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:25,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=92 2023-09-29T06:14:25,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=92 2023-09-29T06:14:25,890 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=92, resume processing ppid=91 2023-09-29T06:14:25,890 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=92, ppid=91, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5830 sec 2023-09-29T06:14:25,892 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=91, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 1.5890 sec 2023-09-29T06:14:26,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on eefd39c60cf58e105fccf6c634940207 2023-09-29T06:14:26,169 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing eefd39c60cf58e105fccf6c634940207 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2023-09-29T06:14:26,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=A 2023-09-29T06:14:26,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:26,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=B 2023-09-29T06:14:26,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:26,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=C 2023-09-29T06:14:26,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:26,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/075e69ca8ec342bcb60fbd4eebd76487 is 50, key is test_row_0/A:col10/1695968066168/Put/seqid=0 2023-09-29T06:14:26,184 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=482 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/075e69ca8ec342bcb60fbd4eebd76487 2023-09-29T06:14:26,192 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/e10adeba67024cf8a8b2128b0ad92ad4 is 50, key is test_row_0/B:col10/1695968066168/Put/seqid=0 2023-09-29T06:14:26,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:26,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968126187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:26,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:26,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968126190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:26,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:26,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968126192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:26,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:26,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968126193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:26,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:26,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968126295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:26,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:26,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968126295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:26,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:26,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968126297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:26,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:26,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968126298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:26,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=91 2023-09-29T06:14:26,417 INFO [Thread-1523 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 91 completed 2023-09-29T06:14:26,419 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:14:26,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2023-09-29T06:14:26,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=93 2023-09-29T06:14:26,432 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:14:26,433 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:14:26,433 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:14:26,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:26,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968126497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:26,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:26,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968126498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:26,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:26,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968126500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:26,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:26,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968126501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:26,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=93 2023-09-29T06:14:26,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2023-09-29T06:14:26,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:26,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. as already flushing 2023-09-29T06:14:26,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:26,594 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(53): pid=94 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:26,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:26,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:26,610 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=482 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/e10adeba67024cf8a8b2128b0ad92ad4 2023-09-29T06:14:26,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/0081a6e57aa640ec85125a8fca0351d7 is 50, key is test_row_0/C:col10/1695968066168/Put/seqid=0 2023-09-29T06:14:26,653 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=482 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/0081a6e57aa640ec85125a8fca0351d7 2023-09-29T06:14:26,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/075e69ca8ec342bcb60fbd4eebd76487 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/075e69ca8ec342bcb60fbd4eebd76487 2023-09-29T06:14:26,661 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/075e69ca8ec342bcb60fbd4eebd76487, entries=200, sequenceid=482, filesize=14.4 K 2023-09-29T06:14:26,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/e10adeba67024cf8a8b2128b0ad92ad4 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/e10adeba67024cf8a8b2128b0ad92ad4 2023-09-29T06:14:26,667 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/e10adeba67024cf8a8b2128b0ad92ad4, entries=150, sequenceid=482, filesize=12.0 K 2023-09-29T06:14:26,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/0081a6e57aa640ec85125a8fca0351d7 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/0081a6e57aa640ec85125a8fca0351d7 2023-09-29T06:14:26,676 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/0081a6e57aa640ec85125a8fca0351d7, entries=150, sequenceid=482, filesize=12.0 K 2023-09-29T06:14:26,677 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for eefd39c60cf58e105fccf6c634940207 in 508ms, sequenceid=482, compaction requested=true 2023-09-29T06:14:26,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:26,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eefd39c60cf58e105fccf6c634940207:A, priority=-2147483648, current under compaction store size is 1 2023-09-29T06:14:26,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:26,677 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:14:26,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eefd39c60cf58e105fccf6c634940207:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:14:26,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:26,677 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:14:26,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eefd39c60cf58e105fccf6c634940207:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:14:26,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:14:26,678 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40348 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:14:26,678 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): eefd39c60cf58e105fccf6c634940207/A is initiating minor compaction (all files) 2023-09-29T06:14:26,678 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of eefd39c60cf58e105fccf6c634940207/A in TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:26,679 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/6b09fd12c1714ea0ac2ea2c664d23588, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/ae6e5eb744e943b58ec086aa8af5f4e5, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/075e69ca8ec342bcb60fbd4eebd76487] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp, totalSize=39.4 K 2023-09-29T06:14:26,679 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37908 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:14:26,679 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): eefd39c60cf58e105fccf6c634940207/B is initiating minor compaction (all files) 2023-09-29T06:14:26,679 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of eefd39c60cf58e105fccf6c634940207/B in TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:26,679 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/761d5954d6154b18aced7e6c37f280aa, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/8eff75d6da144372b0afe79378fbfed0, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/e10adeba67024cf8a8b2128b0ad92ad4] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp, totalSize=37.0 K 2023-09-29T06:14:26,679 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 6b09fd12c1714ea0ac2ea2c664d23588, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1695968063275 2023-09-29T06:14:26,679 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 761d5954d6154b18aced7e6c37f280aa, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1695968063275 2023-09-29T06:14:26,679 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting ae6e5eb744e943b58ec086aa8af5f4e5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1695968063900 2023-09-29T06:14:26,680 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 8eff75d6da144372b0afe79378fbfed0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1695968063900 2023-09-29T06:14:26,680 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 075e69ca8ec342bcb60fbd4eebd76487, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=482, earliestPutTs=1695968065046 2023-09-29T06:14:26,680 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting e10adeba67024cf8a8b2128b0ad92ad4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=482, earliestPutTs=1695968065046 2023-09-29T06:14:26,687 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eefd39c60cf58e105fccf6c634940207#A#compaction#333 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:14:26,688 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/60a45d65e77c4e08918d3a35e801b603 is 50, key is test_row_0/A:col10/1695968066168/Put/seqid=0 2023-09-29T06:14:26,690 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eefd39c60cf58e105fccf6c634940207#B#compaction#334 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:14:26,691 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/bc6799793c684e319ed0a9d899bf7f9a is 50, key is test_row_0/B:col10/1695968066168/Put/seqid=0 2023-09-29T06:14:26,727 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/60a45d65e77c4e08918d3a35e801b603 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/60a45d65e77c4e08918d3a35e801b603 2023-09-29T06:14:26,733 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in eefd39c60cf58e105fccf6c634940207/A of eefd39c60cf58e105fccf6c634940207 into 60a45d65e77c4e08918d3a35e801b603(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:14:26,733 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:26,733 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207., storeName=eefd39c60cf58e105fccf6c634940207/A, priority=13, startTime=1695968066677; duration=0sec 2023-09-29T06:14:26,733 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:14:26,733 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eefd39c60cf58e105fccf6c634940207:A 2023-09-29T06:14:26,733 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:14:26,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=93 2023-09-29T06:14:26,734 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37874 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:14:26,734 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): eefd39c60cf58e105fccf6c634940207/C is initiating minor compaction (all files) 2023-09-29T06:14:26,735 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of eefd39c60cf58e105fccf6c634940207/C in TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:26,735 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/7c1fdce5038a4ecfaf79e455eced7487, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/c0eefdd6d1e440c6aad214de68895a42, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/0081a6e57aa640ec85125a8fca0351d7] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp, totalSize=37.0 K 2023-09-29T06:14:26,735 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 7c1fdce5038a4ecfaf79e455eced7487, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1695968063275 2023-09-29T06:14:26,736 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting c0eefdd6d1e440c6aad214de68895a42, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1695968063900 2023-09-29T06:14:26,736 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 0081a6e57aa640ec85125a8fca0351d7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=482, earliestPutTs=1695968065046 2023-09-29T06:14:26,742 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eefd39c60cf58e105fccf6c634940207#C#compaction#335 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:14:26,743 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/aa010be340d949d993ef50e09d456f2e is 50, key is test_row_0/C:col10/1695968066168/Put/seqid=0 2023-09-29T06:14:26,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2023-09-29T06:14:26,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:26,747 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2819): Flushing eefd39c60cf58e105fccf6c634940207 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2023-09-29T06:14:26,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=A 2023-09-29T06:14:26,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:26,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=B 2023-09-29T06:14:26,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:26,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=C 2023-09-29T06:14:26,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:26,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/9bfe71c285e843d1a65b379b47f2289e is 50, key is test_row_0/A:col10/1695968066175/Put/seqid=0 2023-09-29T06:14:26,766 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/aa010be340d949d993ef50e09d456f2e as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/aa010be340d949d993ef50e09d456f2e 2023-09-29T06:14:26,772 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in eefd39c60cf58e105fccf6c634940207/C of eefd39c60cf58e105fccf6c634940207 into aa010be340d949d993ef50e09d456f2e(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:14:26,772 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:26,772 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207., storeName=eefd39c60cf58e105fccf6c634940207/C, priority=13, startTime=1695968066677; duration=0sec 2023-09-29T06:14:26,773 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:26,773 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eefd39c60cf58e105fccf6c634940207:C 2023-09-29T06:14:26,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. as already flushing 2023-09-29T06:14:26,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on eefd39c60cf58e105fccf6c634940207 2023-09-29T06:14:26,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:26,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968126825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:26,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:26,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968126826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:26,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:26,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968126830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:26,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:26,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968126831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:26,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:26,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968126931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:26,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:26,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968126932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:26,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:26,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968126934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:26,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:26,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968126935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:27,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=93 2023-09-29T06:14:27,125 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/bc6799793c684e319ed0a9d899bf7f9a as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/bc6799793c684e319ed0a9d899bf7f9a 2023-09-29T06:14:27,129 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in eefd39c60cf58e105fccf6c634940207/B of eefd39c60cf58e105fccf6c634940207 into bc6799793c684e319ed0a9d899bf7f9a(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:14:27,129 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:27,130 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207., storeName=eefd39c60cf58e105fccf6c634940207/B, priority=13, startTime=1695968066677; duration=0sec 2023-09-29T06:14:27,130 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:27,130 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eefd39c60cf58e105fccf6c634940207:B 2023-09-29T06:14:27,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:27,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968127135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:27,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:27,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968127137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:27,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:27,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968127138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:27,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 250 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968127138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:27,175 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/9bfe71c285e843d1a65b379b47f2289e 2023-09-29T06:14:27,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/63764cae57d34dd583d65973d6c02e82 is 50, key is test_row_0/B:col10/1695968066175/Put/seqid=0 2023-09-29T06:14:27,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:27,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 251 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41050 deadline: 1695968127439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:27,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:27,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:27,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 251 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41006 deadline: 1695968127439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:27,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41018 deadline: 1695968127440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:27,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:27,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 251 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41062 deadline: 1695968127440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:27,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:27,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:41054 deadline: 1695968127466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:27,471 DEBUG [Thread-1519 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=8, retries=16, started=18235 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_2' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207., hostname=jenkins-hbase3.apache.org,39783,1695967978138, seqNum=2, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:572) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:559) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:197) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:218) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:750) 2023-09-29T06:14:27,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=93 2023-09-29T06:14:27,599 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/63764cae57d34dd583d65973d6c02e82 2023-09-29T06:14:27,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/53b5aad4967b4d1688c01ac2e67537a9 is 50, key is test_row_0/C:col10/1695968066175/Put/seqid=0 2023-09-29T06:14:27,639 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/53b5aad4967b4d1688c01ac2e67537a9 2023-09-29T06:14:27,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/9bfe71c285e843d1a65b379b47f2289e as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/9bfe71c285e843d1a65b379b47f2289e 2023-09-29T06:14:27,659 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/9bfe71c285e843d1a65b379b47f2289e, entries=150, sequenceid=497, filesize=12.0 K 2023-09-29T06:14:27,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/63764cae57d34dd583d65973d6c02e82 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/63764cae57d34dd583d65973d6c02e82 2023-09-29T06:14:27,664 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/63764cae57d34dd583d65973d6c02e82, entries=150, sequenceid=497, filesize=12.0 K 2023-09-29T06:14:27,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/53b5aad4967b4d1688c01ac2e67537a9 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/53b5aad4967b4d1688c01ac2e67537a9 2023-09-29T06:14:27,670 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/53b5aad4967b4d1688c01ac2e67537a9, entries=150, sequenceid=497, filesize=12.0 K 2023-09-29T06:14:27,671 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3022): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for eefd39c60cf58e105fccf6c634940207 in 924ms, sequenceid=497, compaction requested=false 2023-09-29T06:14:27,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2520): Flush status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:27,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:27,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=94 2023-09-29T06:14:27,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=94 2023-09-29T06:14:27,673 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=94, resume processing ppid=93 2023-09-29T06:14:27,673 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=94, ppid=93, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2390 sec 2023-09-29T06:14:27,675 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=93, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 1.2550 sec 2023-09-29T06:14:27,881 DEBUG [Thread-1526 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x55e0ed62 to 127.0.0.1:49344 2023-09-29T06:14:27,881 DEBUG [Thread-1528 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0ce1917b to 127.0.0.1:49344 2023-09-29T06:14:27,881 DEBUG [Thread-1526 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:14:27,881 DEBUG [Thread-1528 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:14:27,883 DEBUG [Thread-1524 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x681d2c78 to 127.0.0.1:49344 2023-09-29T06:14:27,883 DEBUG [Thread-1524 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:14:27,883 DEBUG [Thread-1530 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x6e7b70bf to 127.0.0.1:49344 2023-09-29T06:14:27,883 DEBUG [Thread-1530 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:14:27,884 DEBUG [Thread-1532 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x15adc103 to 127.0.0.1:49344 2023-09-29T06:14:27,884 DEBUG [Thread-1532 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:14:27,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on eefd39c60cf58e105fccf6c634940207 2023-09-29T06:14:27,945 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing eefd39c60cf58e105fccf6c634940207 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2023-09-29T06:14:27,946 DEBUG [Thread-1521 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x7c744127 to 127.0.0.1:49344 2023-09-29T06:14:27,946 DEBUG [Thread-1521 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:14:27,947 DEBUG [Thread-1515 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x71e339c4 to 127.0.0.1:49344 2023-09-29T06:14:27,947 DEBUG [Thread-1515 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:14:27,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=A 2023-09-29T06:14:27,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:27,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=B 2023-09-29T06:14:27,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:27,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=C 2023-09-29T06:14:27,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:27,948 DEBUG [Thread-1517 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x46ffdc41 to 127.0.0.1:49344 2023-09-29T06:14:27,948 DEBUG [Thread-1517 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:14:27,949 DEBUG [Thread-1513 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x5017cdbc to 127.0.0.1:49344 2023-09-29T06:14:27,949 DEBUG [Thread-1513 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:14:27,951 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/1e32b759c8344226aecfc625c91dd428 is 50, key is test_row_0/A:col10/1695968067945/Put/seqid=0 2023-09-29T06:14:28,360 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=525 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/1e32b759c8344226aecfc625c91dd428 2023-09-29T06:14:28,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/2982e1582232408b8d13f6cc27b3e170 is 50, key is test_row_0/B:col10/1695968067945/Put/seqid=0 2023-09-29T06:14:28,371 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=525 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/2982e1582232408b8d13f6cc27b3e170 2023-09-29T06:14:28,377 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/f0af7bf60c2e469aa3999a6baa129d1f is 50, key is test_row_0/C:col10/1695968067945/Put/seqid=0 2023-09-29T06:14:28,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=93 2023-09-29T06:14:28,542 INFO [Thread-1523 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 93 completed 2023-09-29T06:14:28,782 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=525 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/f0af7bf60c2e469aa3999a6baa129d1f 2023-09-29T06:14:28,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/1e32b759c8344226aecfc625c91dd428 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/1e32b759c8344226aecfc625c91dd428 2023-09-29T06:14:28,790 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/1e32b759c8344226aecfc625c91dd428, entries=150, sequenceid=525, filesize=12.0 K 2023-09-29T06:14:28,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/2982e1582232408b8d13f6cc27b3e170 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/2982e1582232408b8d13f6cc27b3e170 2023-09-29T06:14:28,795 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/2982e1582232408b8d13f6cc27b3e170, entries=150, sequenceid=525, filesize=12.0 K 2023-09-29T06:14:28,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/f0af7bf60c2e469aa3999a6baa129d1f as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/f0af7bf60c2e469aa3999a6baa129d1f 2023-09-29T06:14:28,801 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/f0af7bf60c2e469aa3999a6baa129d1f, entries=150, sequenceid=525, filesize=12.0 K 2023-09-29T06:14:28,801 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=6.71 KB/6870 for eefd39c60cf58e105fccf6c634940207 in 856ms, sequenceid=525, compaction requested=true 2023-09-29T06:14:28,802 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:28,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eefd39c60cf58e105fccf6c634940207:A, priority=-2147483648, current under compaction store size is 1 2023-09-29T06:14:28,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:28,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eefd39c60cf58e105fccf6c634940207:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:14:28,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:14:28,802 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:14:28,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eefd39c60cf58e105fccf6c634940207:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:14:28,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2023-09-29T06:14:28,802 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:14:28,803 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38010 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:14:28,803 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38010 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:14:28,803 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): eefd39c60cf58e105fccf6c634940207/B is initiating minor compaction (all files) 2023-09-29T06:14:28,803 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): eefd39c60cf58e105fccf6c634940207/A is initiating minor compaction (all files) 2023-09-29T06:14:28,803 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of eefd39c60cf58e105fccf6c634940207/B in TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:28,803 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of eefd39c60cf58e105fccf6c634940207/A in TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:28,803 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/bc6799793c684e319ed0a9d899bf7f9a, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/63764cae57d34dd583d65973d6c02e82, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/2982e1582232408b8d13f6cc27b3e170] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp, totalSize=37.1 K 2023-09-29T06:14:28,803 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/60a45d65e77c4e08918d3a35e801b603, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/9bfe71c285e843d1a65b379b47f2289e, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/1e32b759c8344226aecfc625c91dd428] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp, totalSize=37.1 K 2023-09-29T06:14:28,804 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 60a45d65e77c4e08918d3a35e801b603, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=482, earliestPutTs=1695968065046 2023-09-29T06:14:28,804 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting bc6799793c684e319ed0a9d899bf7f9a, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=482, earliestPutTs=1695968065046 2023-09-29T06:14:28,804 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 9bfe71c285e843d1a65b379b47f2289e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1695968066175 2023-09-29T06:14:28,804 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 63764cae57d34dd583d65973d6c02e82, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1695968066175 2023-09-29T06:14:28,804 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 1e32b759c8344226aecfc625c91dd428, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=525, earliestPutTs=1695968066824 2023-09-29T06:14:28,804 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 2982e1582232408b8d13f6cc27b3e170, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=525, earliestPutTs=1695968066824 2023-09-29T06:14:28,811 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eefd39c60cf58e105fccf6c634940207#B#compaction#342 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:14:28,811 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/419a04d5f7e44ca2a3c2b8eb15871516 is 50, key is test_row_0/B:col10/1695968067945/Put/seqid=0 2023-09-29T06:14:28,835 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eefd39c60cf58e105fccf6c634940207#A#compaction#343 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:14:28,835 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/3f84c36975bf4e1a8c937b915133dad9 is 50, key is test_row_0/A:col10/1695968067945/Put/seqid=0 2023-09-29T06:14:28,859 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/3f84c36975bf4e1a8c937b915133dad9 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/3f84c36975bf4e1a8c937b915133dad9 2023-09-29T06:14:28,864 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in eefd39c60cf58e105fccf6c634940207/A of eefd39c60cf58e105fccf6c634940207 into 3f84c36975bf4e1a8c937b915133dad9(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:14:28,864 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:28,864 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207., storeName=eefd39c60cf58e105fccf6c634940207/A, priority=13, startTime=1695968068802; duration=0sec 2023-09-29T06:14:28,864 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:14:28,864 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eefd39c60cf58e105fccf6c634940207:A 2023-09-29T06:14:28,865 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:14:28,865 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37976 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:14:28,865 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): eefd39c60cf58e105fccf6c634940207/C is initiating minor compaction (all files) 2023-09-29T06:14:28,866 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of eefd39c60cf58e105fccf6c634940207/C in TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:28,866 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/aa010be340d949d993ef50e09d456f2e, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/53b5aad4967b4d1688c01ac2e67537a9, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/f0af7bf60c2e469aa3999a6baa129d1f] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp, totalSize=37.1 K 2023-09-29T06:14:28,866 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting aa010be340d949d993ef50e09d456f2e, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=482, earliestPutTs=1695968065046 2023-09-29T06:14:28,867 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 53b5aad4967b4d1688c01ac2e67537a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1695968066175 2023-09-29T06:14:28,867 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting f0af7bf60c2e469aa3999a6baa129d1f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=525, earliestPutTs=1695968066824 2023-09-29T06:14:28,879 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eefd39c60cf58e105fccf6c634940207#C#compaction#344 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:14:28,879 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/34daaf3ebd0e4e72a2f94f8ea7851430 is 50, key is test_row_0/C:col10/1695968067945/Put/seqid=0 2023-09-29T06:14:28,886 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/34daaf3ebd0e4e72a2f94f8ea7851430 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/34daaf3ebd0e4e72a2f94f8ea7851430 2023-09-29T06:14:28,890 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in eefd39c60cf58e105fccf6c634940207/C of eefd39c60cf58e105fccf6c634940207 into 34daaf3ebd0e4e72a2f94f8ea7851430(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:14:28,890 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:28,890 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207., storeName=eefd39c60cf58e105fccf6c634940207/C, priority=13, startTime=1695968068802; duration=0sec 2023-09-29T06:14:28,890 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:28,890 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eefd39c60cf58e105fccf6c634940207:C 2023-09-29T06:14:29,239 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/419a04d5f7e44ca2a3c2b8eb15871516 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/419a04d5f7e44ca2a3c2b8eb15871516 2023-09-29T06:14:29,245 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in eefd39c60cf58e105fccf6c634940207/B of eefd39c60cf58e105fccf6c634940207 into 419a04d5f7e44ca2a3c2b8eb15871516(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:14:29,245 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:29,245 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207., storeName=eefd39c60cf58e105fccf6c634940207/B, priority=13, startTime=1695968068802; duration=0sec 2023-09-29T06:14:29,245 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:29,245 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eefd39c60cf58e105fccf6c634940207:B 2023-09-29T06:14:37,554 DEBUG [Thread-1519 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x62eb7ea8 to 127.0.0.1:49344 2023-09-29T06:14:37,554 DEBUG [Thread-1519 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:14:37,554 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2023-09-29T06:14:37,554 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(394): wrote 101 2023-09-29T06:14:37,554 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(394): wrote 100 2023-09-29T06:14:37,554 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(394): wrote 100 2023-09-29T06:14:37,554 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2023-09-29T06:14:37,554 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(394): wrote 100 2023-09-29T06:14:37,554 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(396): Readers: 2023-09-29T06:14:37,555 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(398): read 5810 2023-09-29T06:14:37,555 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(398): read 5873 2023-09-29T06:14:37,555 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(398): read 6011 2023-09-29T06:14:37,555 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(398): read 5773 2023-09-29T06:14:37,555 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(398): read 5818 2023-09-29T06:14:37,555 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2023-09-29T06:14:37,555 INFO [Listener at localhost/42843 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-09-29T06:14:37,555 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x01357904 to 127.0.0.1:49344 2023-09-29T06:14:37,555 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:14:37,556 INFO [Listener at localhost/42843 {}] client.HBaseAdmin$18(967): Started disable of TestAcidGuarantees 2023-09-29T06:14:37,556 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$12(2664): Client=jenkins//172.31.12.81 disable TestAcidGuarantees 2023-09-29T06:14:37,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=95, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=TestAcidGuarantees 2023-09-29T06:14:37,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=95 2023-09-29T06:14:37,562 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968077562"}]},"ts":"1695968077562"} 2023-09-29T06:14:37,563 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2023-09-29T06:14:37,565 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(293): Set TestAcidGuarantees to state=DISABLING 2023-09-29T06:14:37,566 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=eefd39c60cf58e105fccf6c634940207, UNASSIGN}] 2023-09-29T06:14:37,567 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=eefd39c60cf58e105fccf6c634940207, UNASSIGN 2023-09-29T06:14:37,568 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=96 updating hbase:meta row=eefd39c60cf58e105fccf6c634940207, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:37,570 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE; CloseRegionProcedure eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138}] 2023-09-29T06:14:37,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=95 2023-09-29T06:14:37,722 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(119): Close eefd39c60cf58e105fccf6c634940207 2023-09-29T06:14:37,722 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1668): Closing eefd39c60cf58e105fccf6c634940207, disabling compactions & flushes 2023-09-29T06:14:37,722 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1690): Closing region TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:37,722 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1711): Waiting without time limit for close lock on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:37,722 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1778): Acquired close lock on TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. after waiting 0 ms 2023-09-29T06:14:37,722 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1788): Updates disabled for region TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:37,722 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(2819): Flushing eefd39c60cf58e105fccf6c634940207 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2023-09-29T06:14:37,723 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=A 2023-09-29T06:14:37,723 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:37,723 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=B 2023-09-29T06:14:37,723 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:37,723 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.CompactingMemStore(204): FLUSHING TO DISK eefd39c60cf58e105fccf6c634940207, store=C 2023-09-29T06:14:37,723 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:37,728 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/299a5acdd9c04628a9d77f84cc6b6d9b is 50, key is test_row_2/A:col10/1695968077552/Put/seqid=0 2023-09-29T06:14:37,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=95 2023-09-29T06:14:38,132 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=533 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/299a5acdd9c04628a9d77f84cc6b6d9b 2023-09-29T06:14:38,139 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/d923b50bec464fcea60fdb87df94ecd5 is 50, key is test_row_2/B:col10/1695968077552/Put/seqid=0 2023-09-29T06:14:38,151 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=533 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/d923b50bec464fcea60fdb87df94ecd5 2023-09-29T06:14:38,156 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/26374d77d3ca46cba231986c76cd839c is 50, key is test_row_2/C:col10/1695968077552/Put/seqid=0 2023-09-29T06:14:38,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=95 2023-09-29T06:14:38,559 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=533 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/26374d77d3ca46cba231986c76cd839c 2023-09-29T06:14:38,563 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/A/299a5acdd9c04628a9d77f84cc6b6d9b as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/299a5acdd9c04628a9d77f84cc6b6d9b 2023-09-29T06:14:38,567 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/299a5acdd9c04628a9d77f84cc6b6d9b, entries=50, sequenceid=533, filesize=7.2 K 2023-09-29T06:14:38,568 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/B/d923b50bec464fcea60fdb87df94ecd5 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/d923b50bec464fcea60fdb87df94ecd5 2023-09-29T06:14:38,574 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/d923b50bec464fcea60fdb87df94ecd5, entries=50, sequenceid=533, filesize=7.2 K 2023-09-29T06:14:38,577 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/.tmp/C/26374d77d3ca46cba231986c76cd839c as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/26374d77d3ca46cba231986c76cd839c 2023-09-29T06:14:38,583 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/26374d77d3ca46cba231986c76cd839c, entries=50, sequenceid=533, filesize=7.2 K 2023-09-29T06:14:38,584 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(3022): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for eefd39c60cf58e105fccf6c634940207 in 862ms, sequenceid=533, compaction requested=false 2023-09-29T06:14:38,614 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/e724149fa96c47eea4eee887cad53c7e, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/e0c8e0dbeb9544d2bf31a0bd98eaeaa5, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/b556a8fd5c9245cba9973b546ffb669b, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/ab44d8773a994b7d89a08e0ff2b0b864, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/6633ac64727244dfbc57784591c40e9e, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/12f9e929b4f847ec9242c9a134ed0054, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/73b8f06f0b674f158192d0b7f956275d, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/201cc91e94e743d98d401ba2dd5c417b, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/0df407190a3d4d1abf173167527c8a59, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/e1d7734c00084a869e61c9fee9a14a7c, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/e6fe6cdc2e1b4a7aa67422d67cc886a9, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/b4f0c48fb5c7498388cbf2890705e1d6, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/0f80d17246284443b1c8d291ab8bfcb7, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/ddf5c3cb24ea46c1b18c5bea49ba4992, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/813f62168c4346d4995082ed0697d22c, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/f4742bf9ffa14c61be6959892aad810a, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/6d5cca0f542c4d87869e67aaf86935f5, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/add58ffd1a3348979c943c3c2aee8dce, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/f6ab128c2a5043efa34f082436768f34, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/5f1611b55f4643aa8db0edae1eacec70, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/fca517171ae348179d165d92277badbc, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/d4917e280dbf4e4a8c0ed72524533b33, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/ab16378cf1754deb97bd6d4d2d094e0d, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/718a030fd1f043f28aa9f520d9d66521, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/4b0573c5b22d478398bf534c441301c3, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/725d96e5a1a04c5cad1b8b71b558c936, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/fd7a5ec6038e486ba715bacfd17fc5bd, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/0d2272e59f0b4ada8eeaa90d1c1a90ce, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/8a02a89774714f71b7ce1dd3c10034dc, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/7ad6aab2f06d438c99ffe5623e882bcd, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/6b09fd12c1714ea0ac2ea2c664d23588, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/9ce964a4295c4b999ac1452397c93d2f, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/ae6e5eb744e943b58ec086aa8af5f4e5, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/075e69ca8ec342bcb60fbd4eebd76487, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/60a45d65e77c4e08918d3a35e801b603, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/9bfe71c285e843d1a65b379b47f2289e, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/1e32b759c8344226aecfc625c91dd428] to archive 2023-09-29T06:14:38,615 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2023-09-29T06:14:38,617 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/e724149fa96c47eea4eee887cad53c7e to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/e724149fa96c47eea4eee887cad53c7e 2023-09-29T06:14:38,618 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/e0c8e0dbeb9544d2bf31a0bd98eaeaa5 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/e0c8e0dbeb9544d2bf31a0bd98eaeaa5 2023-09-29T06:14:38,619 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/b556a8fd5c9245cba9973b546ffb669b to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/b556a8fd5c9245cba9973b546ffb669b 2023-09-29T06:14:38,620 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/ab44d8773a994b7d89a08e0ff2b0b864 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/ab44d8773a994b7d89a08e0ff2b0b864 2023-09-29T06:14:38,621 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/6633ac64727244dfbc57784591c40e9e to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/6633ac64727244dfbc57784591c40e9e 2023-09-29T06:14:38,623 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/12f9e929b4f847ec9242c9a134ed0054 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/12f9e929b4f847ec9242c9a134ed0054 2023-09-29T06:14:38,625 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/73b8f06f0b674f158192d0b7f956275d to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/73b8f06f0b674f158192d0b7f956275d 2023-09-29T06:14:38,626 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/201cc91e94e743d98d401ba2dd5c417b to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/201cc91e94e743d98d401ba2dd5c417b 2023-09-29T06:14:38,627 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/0df407190a3d4d1abf173167527c8a59 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/0df407190a3d4d1abf173167527c8a59 2023-09-29T06:14:38,629 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/e1d7734c00084a869e61c9fee9a14a7c to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/e1d7734c00084a869e61c9fee9a14a7c 2023-09-29T06:14:38,630 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/e6fe6cdc2e1b4a7aa67422d67cc886a9 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/e6fe6cdc2e1b4a7aa67422d67cc886a9 2023-09-29T06:14:38,631 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/b4f0c48fb5c7498388cbf2890705e1d6 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/b4f0c48fb5c7498388cbf2890705e1d6 2023-09-29T06:14:38,634 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/0f80d17246284443b1c8d291ab8bfcb7 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/0f80d17246284443b1c8d291ab8bfcb7 2023-09-29T06:14:38,635 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/ddf5c3cb24ea46c1b18c5bea49ba4992 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/ddf5c3cb24ea46c1b18c5bea49ba4992 2023-09-29T06:14:38,636 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/813f62168c4346d4995082ed0697d22c to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/813f62168c4346d4995082ed0697d22c 2023-09-29T06:14:38,637 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/f4742bf9ffa14c61be6959892aad810a to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/f4742bf9ffa14c61be6959892aad810a 2023-09-29T06:14:38,639 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/6d5cca0f542c4d87869e67aaf86935f5 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/6d5cca0f542c4d87869e67aaf86935f5 2023-09-29T06:14:38,640 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/add58ffd1a3348979c943c3c2aee8dce to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/add58ffd1a3348979c943c3c2aee8dce 2023-09-29T06:14:38,641 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/f6ab128c2a5043efa34f082436768f34 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/f6ab128c2a5043efa34f082436768f34 2023-09-29T06:14:38,642 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/5f1611b55f4643aa8db0edae1eacec70 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/5f1611b55f4643aa8db0edae1eacec70 2023-09-29T06:14:38,644 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/fca517171ae348179d165d92277badbc to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/fca517171ae348179d165d92277badbc 2023-09-29T06:14:38,645 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/d4917e280dbf4e4a8c0ed72524533b33 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/d4917e280dbf4e4a8c0ed72524533b33 2023-09-29T06:14:38,647 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/ab16378cf1754deb97bd6d4d2d094e0d to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/ab16378cf1754deb97bd6d4d2d094e0d 2023-09-29T06:14:38,648 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/718a030fd1f043f28aa9f520d9d66521 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/718a030fd1f043f28aa9f520d9d66521 2023-09-29T06:14:38,649 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/4b0573c5b22d478398bf534c441301c3 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/4b0573c5b22d478398bf534c441301c3 2023-09-29T06:14:38,650 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/725d96e5a1a04c5cad1b8b71b558c936 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/725d96e5a1a04c5cad1b8b71b558c936 2023-09-29T06:14:38,651 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/fd7a5ec6038e486ba715bacfd17fc5bd to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/fd7a5ec6038e486ba715bacfd17fc5bd 2023-09-29T06:14:38,652 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/0d2272e59f0b4ada8eeaa90d1c1a90ce to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/0d2272e59f0b4ada8eeaa90d1c1a90ce 2023-09-29T06:14:38,653 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/8a02a89774714f71b7ce1dd3c10034dc to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/8a02a89774714f71b7ce1dd3c10034dc 2023-09-29T06:14:38,655 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/7ad6aab2f06d438c99ffe5623e882bcd to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/7ad6aab2f06d438c99ffe5623e882bcd 2023-09-29T06:14:38,657 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/6b09fd12c1714ea0ac2ea2c664d23588 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/6b09fd12c1714ea0ac2ea2c664d23588 2023-09-29T06:14:38,658 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/9ce964a4295c4b999ac1452397c93d2f to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/9ce964a4295c4b999ac1452397c93d2f 2023-09-29T06:14:38,659 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/ae6e5eb744e943b58ec086aa8af5f4e5 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/ae6e5eb744e943b58ec086aa8af5f4e5 2023-09-29T06:14:38,661 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/075e69ca8ec342bcb60fbd4eebd76487 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/075e69ca8ec342bcb60fbd4eebd76487 2023-09-29T06:14:38,662 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/60a45d65e77c4e08918d3a35e801b603 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/60a45d65e77c4e08918d3a35e801b603 2023-09-29T06:14:38,663 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/9bfe71c285e843d1a65b379b47f2289e to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/9bfe71c285e843d1a65b379b47f2289e 2023-09-29T06:14:38,664 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/1e32b759c8344226aecfc625c91dd428 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/1e32b759c8344226aecfc625c91dd428 2023-09-29T06:14:38,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=95 2023-09-29T06:14:38,668 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/813aa866615746fd93a60db92e8044df, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/9f892ef609564eb1b45fdbf811a46fae, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/cc1fd449d57f4409b0ddfae3d32b3264, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/754fb684c7874804b9e34af0d55ed629, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/44ea540e8b574931ae12c754acac4881, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/4ae6025b57644dbc9814cabb8431d023, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/b6ecac7997d6497cb20286229cbb8de9, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/0d70bd70201840e9a213097087402d45, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/00af6fd8e2a84b2c986646f78867a81b, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/ab3f417decf24aa5aa8e5e1eea38de35, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/da2651d237474a68893c8404ebb0ae4e, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/c18bf2fb56ba47008ea38720958940f2, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/ccb6010a791f43b0a6f4f796cc4065ed, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/b0f024219dba4e3f87393c13c7eff821, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/7678041864c84240bd4cf5211970d90c, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/4bc55da3d3ac43049556b842a22ef33c, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/6f5e0bb75c1247eabf6f2421cfd210ce, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/93478b8d68cf4298b16dd9d82a666f44, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/79b0f34119ed4d2f8ee1f593ca872521, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/b5a3035d5a08424b9319f18f231469ea, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/9f675551499d44109d7d8e4db80d4a6a, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/2c9cd09dd3a24b748c6dc960b7f015f6, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/1641b27f8ef54fd98b44f41140a30c78, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/b49d38840372492986af86b75b629221, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/f138c17e87cc4b1dad17ec9ba29cfeb1, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/097b1f03a9ae466fb6fc97ab588554be, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/9e98700d1060442daa5f885643022801, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/ec1b9d53de6c458fa878c195384ea51a, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/cfe0680b839a4c69b29ba596bf204113, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/8f4009e2b9ae4700bfe50a650be78bd5, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/761d5954d6154b18aced7e6c37f280aa, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/a299b104283140e3a42df685b74a2928, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/8eff75d6da144372b0afe79378fbfed0, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/bc6799793c684e319ed0a9d899bf7f9a, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/e10adeba67024cf8a8b2128b0ad92ad4, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/63764cae57d34dd583d65973d6c02e82, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/2982e1582232408b8d13f6cc27b3e170] to archive 2023-09-29T06:14:38,669 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2023-09-29T06:14:38,671 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/813aa866615746fd93a60db92e8044df to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/813aa866615746fd93a60db92e8044df 2023-09-29T06:14:38,672 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/9f892ef609564eb1b45fdbf811a46fae to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/9f892ef609564eb1b45fdbf811a46fae 2023-09-29T06:14:38,673 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/cc1fd449d57f4409b0ddfae3d32b3264 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/cc1fd449d57f4409b0ddfae3d32b3264 2023-09-29T06:14:38,675 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/754fb684c7874804b9e34af0d55ed629 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/754fb684c7874804b9e34af0d55ed629 2023-09-29T06:14:38,677 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/44ea540e8b574931ae12c754acac4881 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/44ea540e8b574931ae12c754acac4881 2023-09-29T06:14:38,678 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/4ae6025b57644dbc9814cabb8431d023 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/4ae6025b57644dbc9814cabb8431d023 2023-09-29T06:14:38,679 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/b6ecac7997d6497cb20286229cbb8de9 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/b6ecac7997d6497cb20286229cbb8de9 2023-09-29T06:14:38,680 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/0d70bd70201840e9a213097087402d45 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/0d70bd70201840e9a213097087402d45 2023-09-29T06:14:38,681 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/00af6fd8e2a84b2c986646f78867a81b to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/00af6fd8e2a84b2c986646f78867a81b 2023-09-29T06:14:38,682 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/ab3f417decf24aa5aa8e5e1eea38de35 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/ab3f417decf24aa5aa8e5e1eea38de35 2023-09-29T06:14:38,683 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/da2651d237474a68893c8404ebb0ae4e to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/da2651d237474a68893c8404ebb0ae4e 2023-09-29T06:14:38,684 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/c18bf2fb56ba47008ea38720958940f2 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/c18bf2fb56ba47008ea38720958940f2 2023-09-29T06:14:38,685 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/ccb6010a791f43b0a6f4f796cc4065ed to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/ccb6010a791f43b0a6f4f796cc4065ed 2023-09-29T06:14:38,686 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/b0f024219dba4e3f87393c13c7eff821 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/b0f024219dba4e3f87393c13c7eff821 2023-09-29T06:14:38,687 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/7678041864c84240bd4cf5211970d90c to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/7678041864c84240bd4cf5211970d90c 2023-09-29T06:14:38,688 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/4bc55da3d3ac43049556b842a22ef33c to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/4bc55da3d3ac43049556b842a22ef33c 2023-09-29T06:14:38,689 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/6f5e0bb75c1247eabf6f2421cfd210ce to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/6f5e0bb75c1247eabf6f2421cfd210ce 2023-09-29T06:14:38,690 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/93478b8d68cf4298b16dd9d82a666f44 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/93478b8d68cf4298b16dd9d82a666f44 2023-09-29T06:14:38,691 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/79b0f34119ed4d2f8ee1f593ca872521 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/79b0f34119ed4d2f8ee1f593ca872521 2023-09-29T06:14:38,693 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/b5a3035d5a08424b9319f18f231469ea to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/b5a3035d5a08424b9319f18f231469ea 2023-09-29T06:14:38,694 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/9f675551499d44109d7d8e4db80d4a6a to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/9f675551499d44109d7d8e4db80d4a6a 2023-09-29T06:14:38,695 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/2c9cd09dd3a24b748c6dc960b7f015f6 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/2c9cd09dd3a24b748c6dc960b7f015f6 2023-09-29T06:14:38,696 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/1641b27f8ef54fd98b44f41140a30c78 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/1641b27f8ef54fd98b44f41140a30c78 2023-09-29T06:14:38,697 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/b49d38840372492986af86b75b629221 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/b49d38840372492986af86b75b629221 2023-09-29T06:14:38,702 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/f138c17e87cc4b1dad17ec9ba29cfeb1 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/f138c17e87cc4b1dad17ec9ba29cfeb1 2023-09-29T06:14:38,704 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/097b1f03a9ae466fb6fc97ab588554be to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/097b1f03a9ae466fb6fc97ab588554be 2023-09-29T06:14:38,705 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/9e98700d1060442daa5f885643022801 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/9e98700d1060442daa5f885643022801 2023-09-29T06:14:38,707 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/ec1b9d53de6c458fa878c195384ea51a to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/ec1b9d53de6c458fa878c195384ea51a 2023-09-29T06:14:38,708 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/cfe0680b839a4c69b29ba596bf204113 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/cfe0680b839a4c69b29ba596bf204113 2023-09-29T06:14:38,710 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/8f4009e2b9ae4700bfe50a650be78bd5 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/8f4009e2b9ae4700bfe50a650be78bd5 2023-09-29T06:14:38,711 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/761d5954d6154b18aced7e6c37f280aa to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/761d5954d6154b18aced7e6c37f280aa 2023-09-29T06:14:38,713 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/a299b104283140e3a42df685b74a2928 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/a299b104283140e3a42df685b74a2928 2023-09-29T06:14:38,714 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/8eff75d6da144372b0afe79378fbfed0 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/8eff75d6da144372b0afe79378fbfed0 2023-09-29T06:14:38,715 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/bc6799793c684e319ed0a9d899bf7f9a to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/bc6799793c684e319ed0a9d899bf7f9a 2023-09-29T06:14:38,716 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/e10adeba67024cf8a8b2128b0ad92ad4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/e10adeba67024cf8a8b2128b0ad92ad4 2023-09-29T06:14:38,717 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/63764cae57d34dd583d65973d6c02e82 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/63764cae57d34dd583d65973d6c02e82 2023-09-29T06:14:38,719 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/2982e1582232408b8d13f6cc27b3e170 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/2982e1582232408b8d13f6cc27b3e170 2023-09-29T06:14:38,722 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/a559532c42054b8fbd9af3c6e16eb3cf, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/ddedb0a61b1f471ab68d8690326d532e, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/ab63331258dd4ae0ab2f9e03f971b50f, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/be616cd901494830b29b778bcf96df45, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/929d6588860645fc81eb8427973bcde5, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/a893683644694f43b8718d37c524f478, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/9aabc69f201946eea45bb68a53123b3b, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/a88e36fd01674157979701afcc7a0b45, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/fffa3f3a0c42454fb63f52a98bf551d4, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/4d73fdb1bd9d494e973f85dedd3a16f6, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/035bfcf178724ab1bf01d0ccb3c7adb4, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/b3a0352185ea429a8c3127ec6ee75e61, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/d198ee62a70e444ea62722d5fc9d83c8, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/919f04a5c7b247868e1f380e2a9637b5, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/d675812fb4de44f9a9c69972a6b24378, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/b8166efea80040b8b84081478e84a5f1, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/e4ddcef096f1475594dd4ce155f80fee, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/c99c5b538c854f74a7c7146a5c41a727, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/841e9b2f5e094b4c9f5dec742923aaeb, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/248aa285819245b3a2ad2fc69b8db9d1, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/1189e7fc917c425b83efb7168df21020, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/8e5955960bf9465eb12e6a5a28f4b520, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/bdaa8eea0fa8493d976d1e35dd398d1f, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/4ab930775417401ca23b2c50902eb32c, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/d47be896ae06447aa77ba20810b9dac2, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/7606469a95884e0aa4563a5b61029eb7, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/f4aef31651344ae6905da169d4d9d459, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/d3054ed7a8a44c3f826e32a5dbed42ac, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/065c5607fb2146ddb2148dba6dfe8cf5, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/7c1fdce5038a4ecfaf79e455eced7487, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/083688467a384d128b298ad364a697b4, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/c0eefdd6d1e440c6aad214de68895a42, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/aa010be340d949d993ef50e09d456f2e, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/0081a6e57aa640ec85125a8fca0351d7, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/53b5aad4967b4d1688c01ac2e67537a9, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/f0af7bf60c2e469aa3999a6baa129d1f] to archive 2023-09-29T06:14:38,723 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2023-09-29T06:14:38,725 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/a559532c42054b8fbd9af3c6e16eb3cf to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/a559532c42054b8fbd9af3c6e16eb3cf 2023-09-29T06:14:38,727 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/ddedb0a61b1f471ab68d8690326d532e to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/ddedb0a61b1f471ab68d8690326d532e 2023-09-29T06:14:38,728 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/ab63331258dd4ae0ab2f9e03f971b50f to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/ab63331258dd4ae0ab2f9e03f971b50f 2023-09-29T06:14:38,729 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/be616cd901494830b29b778bcf96df45 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/be616cd901494830b29b778bcf96df45 2023-09-29T06:14:38,730 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/929d6588860645fc81eb8427973bcde5 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/929d6588860645fc81eb8427973bcde5 2023-09-29T06:14:38,731 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/a893683644694f43b8718d37c524f478 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/a893683644694f43b8718d37c524f478 2023-09-29T06:14:38,732 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/9aabc69f201946eea45bb68a53123b3b to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/9aabc69f201946eea45bb68a53123b3b 2023-09-29T06:14:38,733 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/a88e36fd01674157979701afcc7a0b45 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/a88e36fd01674157979701afcc7a0b45 2023-09-29T06:14:38,734 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/fffa3f3a0c42454fb63f52a98bf551d4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/fffa3f3a0c42454fb63f52a98bf551d4 2023-09-29T06:14:38,735 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/4d73fdb1bd9d494e973f85dedd3a16f6 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/4d73fdb1bd9d494e973f85dedd3a16f6 2023-09-29T06:14:38,736 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/035bfcf178724ab1bf01d0ccb3c7adb4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/035bfcf178724ab1bf01d0ccb3c7adb4 2023-09-29T06:14:38,737 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/b3a0352185ea429a8c3127ec6ee75e61 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/b3a0352185ea429a8c3127ec6ee75e61 2023-09-29T06:14:38,739 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/d198ee62a70e444ea62722d5fc9d83c8 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/d198ee62a70e444ea62722d5fc9d83c8 2023-09-29T06:14:38,740 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/919f04a5c7b247868e1f380e2a9637b5 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/919f04a5c7b247868e1f380e2a9637b5 2023-09-29T06:14:38,742 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/d675812fb4de44f9a9c69972a6b24378 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/d675812fb4de44f9a9c69972a6b24378 2023-09-29T06:14:38,743 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/b8166efea80040b8b84081478e84a5f1 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/b8166efea80040b8b84081478e84a5f1 2023-09-29T06:14:38,744 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/e4ddcef096f1475594dd4ce155f80fee to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/e4ddcef096f1475594dd4ce155f80fee 2023-09-29T06:14:38,745 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/c99c5b538c854f74a7c7146a5c41a727 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/c99c5b538c854f74a7c7146a5c41a727 2023-09-29T06:14:38,746 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/841e9b2f5e094b4c9f5dec742923aaeb to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/841e9b2f5e094b4c9f5dec742923aaeb 2023-09-29T06:14:38,747 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/248aa285819245b3a2ad2fc69b8db9d1 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/248aa285819245b3a2ad2fc69b8db9d1 2023-09-29T06:14:38,748 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/1189e7fc917c425b83efb7168df21020 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/1189e7fc917c425b83efb7168df21020 2023-09-29T06:14:38,749 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/8e5955960bf9465eb12e6a5a28f4b520 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/8e5955960bf9465eb12e6a5a28f4b520 2023-09-29T06:14:38,750 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/bdaa8eea0fa8493d976d1e35dd398d1f to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/bdaa8eea0fa8493d976d1e35dd398d1f 2023-09-29T06:14:38,751 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/4ab930775417401ca23b2c50902eb32c to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/4ab930775417401ca23b2c50902eb32c 2023-09-29T06:14:38,752 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/d47be896ae06447aa77ba20810b9dac2 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/d47be896ae06447aa77ba20810b9dac2 2023-09-29T06:14:38,753 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/7606469a95884e0aa4563a5b61029eb7 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/7606469a95884e0aa4563a5b61029eb7 2023-09-29T06:14:38,755 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/f4aef31651344ae6905da169d4d9d459 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/f4aef31651344ae6905da169d4d9d459 2023-09-29T06:14:38,756 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/d3054ed7a8a44c3f826e32a5dbed42ac to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/d3054ed7a8a44c3f826e32a5dbed42ac 2023-09-29T06:14:38,757 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/065c5607fb2146ddb2148dba6dfe8cf5 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/065c5607fb2146ddb2148dba6dfe8cf5 2023-09-29T06:14:38,758 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/7c1fdce5038a4ecfaf79e455eced7487 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/7c1fdce5038a4ecfaf79e455eced7487 2023-09-29T06:14:38,759 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/083688467a384d128b298ad364a697b4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/083688467a384d128b298ad364a697b4 2023-09-29T06:14:38,760 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/c0eefdd6d1e440c6aad214de68895a42 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/c0eefdd6d1e440c6aad214de68895a42 2023-09-29T06:14:38,761 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/aa010be340d949d993ef50e09d456f2e to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/aa010be340d949d993ef50e09d456f2e 2023-09-29T06:14:38,762 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/0081a6e57aa640ec85125a8fca0351d7 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/0081a6e57aa640ec85125a8fca0351d7 2023-09-29T06:14:38,764 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/53b5aad4967b4d1688c01ac2e67537a9 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/53b5aad4967b4d1688c01ac2e67537a9 2023-09-29T06:14:38,765 DEBUG [StoreCloser-TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/f0af7bf60c2e469aa3999a6baa129d1f to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/f0af7bf60c2e469aa3999a6baa129d1f 2023-09-29T06:14:38,769 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/recovered.edits/536.seqid, newMaxSeqId=536, maxSeqId=1 2023-09-29T06:14:38,771 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1909): Closed TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207. 2023-09-29T06:14:38,771 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1622): Region close journal for eefd39c60cf58e105fccf6c634940207: 2023-09-29T06:14:38,772 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(163): Closed eefd39c60cf58e105fccf6c634940207 2023-09-29T06:14:38,772 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=96 updating hbase:meta row=eefd39c60cf58e105fccf6c634940207, regionState=CLOSED 2023-09-29T06:14:38,774 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=97, resume processing ppid=96 2023-09-29T06:14:38,774 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=97, ppid=96, state=SUCCESS; CloseRegionProcedure eefd39c60cf58e105fccf6c634940207, server=jenkins-hbase3.apache.org,39783,1695967978138 in 1.2030 sec 2023-09-29T06:14:38,775 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=96, resume processing ppid=95 2023-09-29T06:14:38,776 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=96, ppid=95, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=eefd39c60cf58e105fccf6c634940207, UNASSIGN in 1.2080 sec 2023-09-29T06:14:38,776 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968078776"}]},"ts":"1695968078776"} 2023-09-29T06:14:38,777 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2023-09-29T06:14:38,779 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(305): Set TestAcidGuarantees to state=DISABLED 2023-09-29T06:14:38,781 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=95, state=SUCCESS; DisableTableProcedure table=TestAcidGuarantees in 1.2220 sec 2023-09-29T06:14:39,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=95 2023-09-29T06:14:39,667 INFO [Listener at localhost/42843 {}] client.HBaseAdmin$TableFuture(3677): Operation: DISABLE, Table Name: default:TestAcidGuarantees, procId: 95 completed 2023-09-29T06:14:39,667 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$5(2448): Client=jenkins//172.31.12.81 delete TestAcidGuarantees 2023-09-29T06:14:39,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=98, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=TestAcidGuarantees 2023-09-29T06:14:39,670 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=98, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2023-09-29T06:14:39,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=98 2023-09-29T06:14:39,670 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=98, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2023-09-29T06:14:39,673 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207 2023-09-29T06:14:39,676 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A, FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B, FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C, FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/recovered.edits] 2023-09-29T06:14:39,679 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/299a5acdd9c04628a9d77f84cc6b6d9b to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/299a5acdd9c04628a9d77f84cc6b6d9b 2023-09-29T06:14:39,680 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/3f84c36975bf4e1a8c937b915133dad9 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/A/3f84c36975bf4e1a8c937b915133dad9 2023-09-29T06:14:39,682 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/419a04d5f7e44ca2a3c2b8eb15871516 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/419a04d5f7e44ca2a3c2b8eb15871516 2023-09-29T06:14:39,684 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/d923b50bec464fcea60fdb87df94ecd5 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/B/d923b50bec464fcea60fdb87df94ecd5 2023-09-29T06:14:39,688 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/26374d77d3ca46cba231986c76cd839c to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/26374d77d3ca46cba231986c76cd839c 2023-09-29T06:14:39,689 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/34daaf3ebd0e4e72a2f94f8ea7851430 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/C/34daaf3ebd0e4e72a2f94f8ea7851430 2023-09-29T06:14:39,694 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/recovered.edits/536.seqid to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207/recovered.edits/536.seqid 2023-09-29T06:14:39,694 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/eefd39c60cf58e105fccf6c634940207 2023-09-29T06:14:39,695 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(293): Archived TestAcidGuarantees regions 2023-09-29T06:14:39,696 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=98, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2023-09-29T06:14:39,699 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2023-09-29T06:14:39,703 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(387): Removing 'TestAcidGuarantees' descriptor. 2023-09-29T06:14:39,704 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=98, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2023-09-29T06:14:39,704 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(377): Removing 'TestAcidGuarantees' from region states. 2023-09-29T06:14:39,704 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1695968079704"}]},"ts":"9223372036854775807"} 2023-09-29T06:14:39,706 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-09-29T06:14:39,706 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => eefd39c60cf58e105fccf6c634940207, NAME => 'TestAcidGuarantees,,1695968046671.eefd39c60cf58e105fccf6c634940207.', STARTKEY => '', ENDKEY => ''}] 2023-09-29T06:14:39,706 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(381): Marking 'TestAcidGuarantees' as deleted. 2023-09-29T06:14:39,706 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1695968079706"}]},"ts":"9223372036854775807"} 2023-09-29T06:14:39,710 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1678): Deleted table TestAcidGuarantees state from META 2023-09-29T06:14:39,713 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(127): Finished pid=98, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2023-09-29T06:14:39,714 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=98, state=SUCCESS; DeleteTableProcedure table=TestAcidGuarantees in 46 msec 2023-09-29T06:14:39,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=98 2023-09-29T06:14:39,772 INFO [Listener at localhost/42843 {}] client.HBaseAdmin$TableFuture(3677): Operation: DELETE, Table Name: default:TestAcidGuarantees, procId: 98 completed 2023-09-29T06:14:39,785 INFO [Listener at localhost/42843 {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithAdaptivePolicy#testGetAtomicity Thread=218 (was 218), OpenFileDescriptor=541 (was 562), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=777 (was 712) - SystemLoadAverage LEAK? -, ProcessCount=171 (was 171), AvailableMemoryMB=1532 (was 1814) 2023-09-29T06:14:39,800 INFO [Listener at localhost/42843 {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithAdaptivePolicy#testMobScanAtomicity Thread=218, OpenFileDescriptor=541, MaxFileDescriptor=60000, SystemLoadAverage=777, ProcessCount=171, AvailableMemoryMB=1532 2023-09-29T06:14:39,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] util.TableDescriptorChecker(342): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2023-09-29T06:14:39,802 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'ADAPTIVE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-29T06:14:39,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=99, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestAcidGuarantees 2023-09-29T06:14:39,804 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=99, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2023-09-29T06:14:39,805 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:14:39,805 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 99 2023-09-29T06:14:39,806 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=99, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-29T06:14:39,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=99 2023-09-29T06:14:39,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=99 2023-09-29T06:14:40,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=99 2023-09-29T06:14:40,221 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'ADAPTIVE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a 2023-09-29T06:14:40,227 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(888): Instantiated TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:14:40,227 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1668): Closing 18784ff25b183a4d3733801ace5c9d09, disabling compactions & flushes 2023-09-29T06:14:40,227 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1690): Closing region TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:40,227 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:40,227 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. after waiting 0 ms 2023-09-29T06:14:40,227 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:40,227 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1909): Closed TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:40,227 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1622): Region close journal for 18784ff25b183a4d3733801ace5c9d09: 2023-09-29T06:14:40,228 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=99, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2023-09-29T06:14:40,228 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1695968080228"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1695968080228"}]},"ts":"1695968080228"} 2023-09-29T06:14:40,229 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-29T06:14:40,230 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=99, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-29T06:14:40,230 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968080230"}]},"ts":"1695968080230"} 2023-09-29T06:14:40,231 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2023-09-29T06:14:40,236 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=18784ff25b183a4d3733801ace5c9d09, ASSIGN}] 2023-09-29T06:14:40,238 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=18784ff25b183a4d3733801ace5c9d09, ASSIGN 2023-09-29T06:14:40,239 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=18784ff25b183a4d3733801ace5c9d09, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,39783,1695967978138; forceNewPlan=false, retain=false 2023-09-29T06:14:40,390 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=100 updating hbase:meta row=18784ff25b183a4d3733801ace5c9d09, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:40,391 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=101, ppid=100, state=RUNNABLE; OpenRegionProcedure 18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138}] 2023-09-29T06:14:40,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=99 2023-09-29T06:14:40,546 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] handler.AssignRegionHandler(130): Open TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:40,546 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(7254): Opening region: {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} 2023-09-29T06:14:40,546 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:14:40,547 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(888): Instantiated TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:14:40,547 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(7296): checking encryption for 18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:14:40,547 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(7299): checking classloading for 18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:14:40,548 INFO [StoreOpener-18784ff25b183a4d3733801ace5c9d09-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:14:40,549 INFO [StoreOpener-18784ff25b183a4d3733801ace5c9d09-1 {}] regionserver.CompactingMemStore(121): Store=A, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2023-09-29T06:14:40,550 INFO [StoreOpener-18784ff25b183a4d3733801ace5c9d09-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 18784ff25b183a4d3733801ace5c9d09 columnFamilyName A 2023-09-29T06:14:40,550 DEBUG [StoreOpener-18784ff25b183a4d3733801ace5c9d09-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:14:40,552 INFO [StoreOpener-18784ff25b183a4d3733801ace5c9d09-1 {}] regionserver.HStore(324): Store=18784ff25b183a4d3733801ace5c9d09/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:14:40,552 INFO [StoreOpener-18784ff25b183a4d3733801ace5c9d09-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:14:40,553 INFO [StoreOpener-18784ff25b183a4d3733801ace5c9d09-1 {}] regionserver.CompactingMemStore(121): Store=B, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2023-09-29T06:14:40,553 INFO [StoreOpener-18784ff25b183a4d3733801ace5c9d09-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 18784ff25b183a4d3733801ace5c9d09 columnFamilyName B 2023-09-29T06:14:40,553 DEBUG [StoreOpener-18784ff25b183a4d3733801ace5c9d09-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:14:40,553 INFO [StoreOpener-18784ff25b183a4d3733801ace5c9d09-1 {}] regionserver.HStore(324): Store=18784ff25b183a4d3733801ace5c9d09/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:14:40,553 INFO [StoreOpener-18784ff25b183a4d3733801ace5c9d09-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:14:40,558 INFO [StoreOpener-18784ff25b183a4d3733801ace5c9d09-1 {}] regionserver.CompactingMemStore(121): Store=C, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2023-09-29T06:14:40,558 INFO [StoreOpener-18784ff25b183a4d3733801ace5c9d09-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 18784ff25b183a4d3733801ace5c9d09 columnFamilyName C 2023-09-29T06:14:40,558 DEBUG [StoreOpener-18784ff25b183a4d3733801ace5c9d09-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:14:40,559 INFO [StoreOpener-18784ff25b183a4d3733801ace5c9d09-1 {}] regionserver.HStore(324): Store=18784ff25b183a4d3733801ace5c9d09/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:14:40,559 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(1169): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:40,559 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:14:40,560 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:14:40,561 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2023-09-29T06:14:40,563 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(1076): writing seq id for 18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:14:40,564 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-29T06:14:40,565 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(1093): Opened 18784ff25b183a4d3733801ace5c9d09; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67299370, jitterRate=0.0028387606143951416}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2023-09-29T06:14:40,566 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(992): Region open journal for 18784ff25b183a4d3733801ace5c9d09: 2023-09-29T06:14:40,566 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegionServer(2556): Post open deploy tasks for TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09., pid=101, masterSystemTime=1695968080543 2023-09-29T06:14:40,568 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegionServer(2583): Finished post open deploy task for TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:40,568 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] handler.AssignRegionHandler(158): Opened TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:40,568 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=100 updating hbase:meta row=18784ff25b183a4d3733801ace5c9d09, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:40,570 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=101, resume processing ppid=100 2023-09-29T06:14:40,570 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=101, ppid=100, state=SUCCESS; OpenRegionProcedure 18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 in 178 msec 2023-09-29T06:14:40,572 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=100, resume processing ppid=99 2023-09-29T06:14:40,572 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=100, ppid=99, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=18784ff25b183a4d3733801ace5c9d09, ASSIGN in 334 msec 2023-09-29T06:14:40,572 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=99, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-29T06:14:40,573 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968080572"}]},"ts":"1695968080572"} 2023-09-29T06:14:40,573 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2023-09-29T06:14:40,577 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=99, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2023-09-29T06:14:40,578 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=99, state=SUCCESS; CreateTableProcedure table=TestAcidGuarantees in 775 msec 2023-09-29T06:14:40,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=99 2023-09-29T06:14:40,910 INFO [Listener at localhost/42843 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: default:TestAcidGuarantees, procId: 99 completed 2023-09-29T06:14:40,912 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x4fd4369c to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:14:40,935 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@737ea371, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:14:40,936 DEBUG [Listener at localhost/42843 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:14:40,938 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:52088, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:14:40,939 DEBUG [Listener at localhost/42843 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-29T06:14:40,940 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:44646, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-29T06:14:40,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] util.TableDescriptorChecker(342): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2023-09-29T06:14:40,942 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$13(2700): Client=jenkins//172.31.12.81 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'ADAPTIVE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'ADAPTIVE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-29T06:14:40,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=102, state=RUNNABLE:MODIFY_TABLE_PREPARE; ModifyTableProcedure table=TestAcidGuarantees 2023-09-29T06:14:40,963 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.963 2023-09-29T06:14:40,963 INFO [PEWorker-4 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.999 2023-09-29T06:14:40,965 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=103, ppid=102, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2023-09-29T06:14:40,967 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=18784ff25b183a4d3733801ace5c9d09, REOPEN/MOVE}] 2023-09-29T06:14:40,968 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=104, ppid=103, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=18784ff25b183a4d3733801ace5c9d09, REOPEN/MOVE 2023-09-29T06:14:40,969 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=104 updating hbase:meta row=18784ff25b183a4d3733801ace5c9d09, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:40,970 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=105, ppid=104, state=RUNNABLE; CloseRegionProcedure 18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138}] 2023-09-29T06:14:41,122 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] handler.UnassignRegionHandler(119): Close 18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:14:41,123 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] regionserver.HRegion(1668): Closing 18784ff25b183a4d3733801ace5c9d09, disabling compactions & flushes 2023-09-29T06:14:41,123 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] regionserver.HRegion(1690): Closing region TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:41,123 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] regionserver.HRegion(1711): Waiting without time limit for close lock on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:41,123 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] regionserver.HRegion(1778): Acquired close lock on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. after waiting 0 ms 2023-09-29T06:14:41,123 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] regionserver.HRegion(1788): Updates disabled for region TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:41,131 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-09-29T06:14:41,131 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] regionserver.HRegion(1909): Closed TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:41,131 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] regionserver.HRegion(1622): Region close journal for 18784ff25b183a4d3733801ace5c9d09: 2023-09-29T06:14:41,131 WARN [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] regionserver.HRegionServer(3735): Not adding moved region record: 18784ff25b183a4d3733801ace5c9d09 to self. 2023-09-29T06:14:41,133 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] handler.UnassignRegionHandler(163): Closed 18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:14:41,133 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=104 updating hbase:meta row=18784ff25b183a4d3733801ace5c9d09, regionState=CLOSED 2023-09-29T06:14:41,136 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=105, resume processing ppid=104 2023-09-29T06:14:41,136 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=105, ppid=104, state=SUCCESS; CloseRegionProcedure 18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 in 165 msec 2023-09-29T06:14:41,137 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=104, ppid=103, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=18784ff25b183a4d3733801ace5c9d09, REOPEN/MOVE; state=CLOSED, location=jenkins-hbase3.apache.org,39783,1695967978138; forceNewPlan=false, retain=true 2023-09-29T06:14:41,287 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=104 updating hbase:meta row=18784ff25b183a4d3733801ace5c9d09, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:41,289 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=106, ppid=104, state=RUNNABLE; OpenRegionProcedure 18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138}] 2023-09-29T06:14:41,444 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] handler.AssignRegionHandler(130): Open TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:41,444 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(7254): Opening region: {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} 2023-09-29T06:14:41,445 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:14:41,445 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(888): Instantiated TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:14:41,445 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(7296): checking encryption for 18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:14:41,445 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(7299): checking classloading for 18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:14:41,446 INFO [StoreOpener-18784ff25b183a4d3733801ace5c9d09-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:14:41,447 INFO [StoreOpener-18784ff25b183a4d3733801ace5c9d09-1 {}] regionserver.CompactingMemStore(121): Store=A, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2023-09-29T06:14:41,448 INFO [StoreOpener-18784ff25b183a4d3733801ace5c9d09-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 18784ff25b183a4d3733801ace5c9d09 columnFamilyName A 2023-09-29T06:14:41,449 DEBUG [StoreOpener-18784ff25b183a4d3733801ace5c9d09-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:14:41,450 INFO [StoreOpener-18784ff25b183a4d3733801ace5c9d09-1 {}] regionserver.HStore(324): Store=18784ff25b183a4d3733801ace5c9d09/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:14:41,450 INFO [StoreOpener-18784ff25b183a4d3733801ace5c9d09-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:14:41,451 INFO [StoreOpener-18784ff25b183a4d3733801ace5c9d09-1 {}] regionserver.CompactingMemStore(121): Store=B, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2023-09-29T06:14:41,451 INFO [StoreOpener-18784ff25b183a4d3733801ace5c9d09-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 18784ff25b183a4d3733801ace5c9d09 columnFamilyName B 2023-09-29T06:14:41,451 DEBUG [StoreOpener-18784ff25b183a4d3733801ace5c9d09-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:14:41,451 INFO [StoreOpener-18784ff25b183a4d3733801ace5c9d09-1 {}] regionserver.HStore(324): Store=18784ff25b183a4d3733801ace5c9d09/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:14:41,452 INFO [StoreOpener-18784ff25b183a4d3733801ace5c9d09-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:14:41,452 INFO [StoreOpener-18784ff25b183a4d3733801ace5c9d09-1 {}] regionserver.CompactingMemStore(121): Store=C, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2023-09-29T06:14:41,452 INFO [StoreOpener-18784ff25b183a4d3733801ace5c9d09-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 18784ff25b183a4d3733801ace5c9d09 columnFamilyName C 2023-09-29T06:14:41,452 DEBUG [StoreOpener-18784ff25b183a4d3733801ace5c9d09-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:14:41,453 INFO [StoreOpener-18784ff25b183a4d3733801ace5c9d09-1 {}] regionserver.HStore(324): Store=18784ff25b183a4d3733801ace5c9d09/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:14:41,453 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1169): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:41,454 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:14:41,455 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:14:41,456 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2023-09-29T06:14:41,458 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1076): writing seq id for 18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:14:41,459 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1093): Opened 18784ff25b183a4d3733801ace5c9d09; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62808985, jitterRate=-0.06407319009304047}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2023-09-29T06:14:41,459 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(992): Region open journal for 18784ff25b183a4d3733801ace5c9d09: 2023-09-29T06:14:41,460 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegionServer(2556): Post open deploy tasks for TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09., pid=106, masterSystemTime=1695968081440 2023-09-29T06:14:41,462 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegionServer(2583): Finished post open deploy task for TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:41,462 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] handler.AssignRegionHandler(158): Opened TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:41,462 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=104 updating hbase:meta row=18784ff25b183a4d3733801ace5c9d09, regionState=OPEN, openSeqNum=5, regionLocation=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:41,465 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=106, resume processing ppid=104 2023-09-29T06:14:41,465 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=106, ppid=104, state=SUCCESS; OpenRegionProcedure 18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 in 175 msec 2023-09-29T06:14:41,466 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=104, resume processing ppid=103 2023-09-29T06:14:41,466 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=104, ppid=103, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=18784ff25b183a4d3733801ace5c9d09, REOPEN/MOVE in 498 msec 2023-09-29T06:14:41,468 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=103, resume processing ppid=102 2023-09-29T06:14:41,468 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=103, ppid=102, state=SUCCESS; ReopenTableRegionsProcedure table=TestAcidGuarantees in 502 msec 2023-09-29T06:14:41,469 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=102, state=SUCCESS; ModifyTableProcedure table=TestAcidGuarantees in 526 msec 2023-09-29T06:14:41,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=102 2023-09-29T06:14:41,472 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x405600e8 to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:14:41,477 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ed8bea8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:14:41,478 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x158c1b4d to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:14:41,483 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@652d1854, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:14:41,486 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x798e79ee to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:14:41,496 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c2d43e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:14:41,497 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x51c9bce9 to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:14:41,526 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b02fe37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:14:41,527 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x444cbd8a to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:14:41,532 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@587e7f1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:14:41,533 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x3cf858b9 to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:14:41,540 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@175546b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:14:41,541 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x581a0a23 to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:14:41,553 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2659ac09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:14:41,554 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x3c59e028 to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:14:41,563 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@349aae92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:14:41,564 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x65c2b0e0 to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:14:41,573 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16e954d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:14:41,574 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x6b9d6aa4 to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:14:41,589 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b288287, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:14:41,618 DEBUG [hconnection-0x335edf19-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:14:41,618 DEBUG [hconnection-0x549ecc68-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:14:41,620 DEBUG [hconnection-0x46720c63-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:14:41,621 DEBUG [hconnection-0x539731d1-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:14:41,621 DEBUG [hconnection-0x37085402-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:14:41,622 DEBUG [hconnection-0x181e5acc-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:14:41,622 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:52114, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:14:41,622 DEBUG [hconnection-0x2650665b-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:14:41,623 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:14:41,624 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:52146, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:14:41,624 DEBUG [hconnection-0x6e7b3424-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:14:41,624 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:52104, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:14:41,624 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:52110, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:14:41,624 DEBUG [hconnection-0x5b4eb340-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:14:41,624 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:52138, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:14:41,624 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:52124, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:14:41,625 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:52170, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:14:41,627 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:52152, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:14:41,628 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:52168, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:14:41,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2023-09-29T06:14:41,639 DEBUG [hconnection-0x2c161fdb-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:14:41,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=107 2023-09-29T06:14:41,642 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:52174, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:14:41,644 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:14:41,644 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:14:41,644 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:14:41,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:14:41,675 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 18784ff25b183a4d3733801ace5c9d09 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2023-09-29T06:14:41,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 18784ff25b183a4d3733801ace5c9d09, store=A 2023-09-29T06:14:41,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:41,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 18784ff25b183a4d3733801ace5c9d09, store=B 2023-09-29T06:14:41,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:41,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 18784ff25b183a4d3733801ace5c9d09, store=C 2023-09-29T06:14:41,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:41,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:41,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52114 deadline: 1695968141693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:41,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:41,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968141694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:41,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:41,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52146 deadline: 1695968141694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:41,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:41,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968141694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:41,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:41,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52174 deadline: 1695968141702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:41,727 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309299bf7b19785394e269eff3343c4cd7b7a_18784ff25b183a4d3733801ace5c9d09 is 50, key is test_row_0/A:col10/1695968081666/Put/seqid=0 2023-09-29T06:14:41,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=107 2023-09-29T06:14:41,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:41,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968141795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:41,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:41,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52114 deadline: 1695968141795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:41,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:41,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:41,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52146 deadline: 1695968141795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:41,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968141796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:41,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:41,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52174 deadline: 1695968141806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:41,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2023-09-29T06:14:41,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:41,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. as already flushing 2023-09-29T06:14:41,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:41,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(53): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:41,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:41,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:41,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=107 2023-09-29T06:14:41,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2023-09-29T06:14:41,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:41,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. as already flushing 2023-09-29T06:14:41,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:41,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(53): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:41,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:41,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:41,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:41,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968141999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:42,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:42,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52114 deadline: 1695968142000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:42,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:42,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968142000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:42,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:42,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52146 deadline: 1695968142002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:42,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:42,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52174 deadline: 1695968142008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:42,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2023-09-29T06:14:42,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:42,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. as already flushing 2023-09-29T06:14:42,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:42,119 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(53): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:42,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:42,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:42,160 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309299bf7b19785394e269eff3343c4cd7b7a_18784ff25b183a4d3733801ace5c9d09 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309299bf7b19785394e269eff3343c4cd7b7a_18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:14:42,161 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/3f28254d3ffb4eeb807f5265d15371f3, store: [table=TestAcidGuarantees family=A region=18784ff25b183a4d3733801ace5c9d09] 2023-09-29T06:14:42,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/3f28254d3ffb4eeb807f5265d15371f3 is 175, key is test_row_0/A:col10/1695968081666/Put/seqid=0 2023-09-29T06:14:42,173 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=18, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/3f28254d3ffb4eeb807f5265d15371f3 2023-09-29T06:14:42,199 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/5300f1bc83354dd9ae4085650173502b is 50, key is test_row_0/B:col10/1695968081666/Put/seqid=0 2023-09-29T06:14:42,223 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/5300f1bc83354dd9ae4085650173502b 2023-09-29T06:14:42,240 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/1d141bd1a12241b3abbca77fd2290c24 is 50, key is test_row_0/C:col10/1695968081666/Put/seqid=0 2023-09-29T06:14:42,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=107 2023-09-29T06:14:42,256 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/1d141bd1a12241b3abbca77fd2290c24 2023-09-29T06:14:42,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/3f28254d3ffb4eeb807f5265d15371f3 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/3f28254d3ffb4eeb807f5265d15371f3 2023-09-29T06:14:42,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2023-09-29T06:14:42,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:42,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. as already flushing 2023-09-29T06:14:42,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:42,272 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(53): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:42,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:42,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:42,274 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/3f28254d3ffb4eeb807f5265d15371f3, entries=150, sequenceid=18, filesize=30.2 K 2023-09-29T06:14:42,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/5300f1bc83354dd9ae4085650173502b as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/5300f1bc83354dd9ae4085650173502b 2023-09-29T06:14:42,280 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/5300f1bc83354dd9ae4085650173502b, entries=150, sequenceid=18, filesize=11.7 K 2023-09-29T06:14:42,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/1d141bd1a12241b3abbca77fd2290c24 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/1d141bd1a12241b3abbca77fd2290c24 2023-09-29T06:14:42,285 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/1d141bd1a12241b3abbca77fd2290c24, entries=150, sequenceid=18, filesize=11.7 K 2023-09-29T06:14:42,286 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 18784ff25b183a4d3733801ace5c9d09 in 611ms, sequenceid=18, compaction requested=false 2023-09-29T06:14:42,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 18784ff25b183a4d3733801ace5c9d09: 2023-09-29T06:14:42,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:14:42,304 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 18784ff25b183a4d3733801ace5c9d09 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2023-09-29T06:14:42,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 18784ff25b183a4d3733801ace5c9d09, store=A 2023-09-29T06:14:42,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:42,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 18784ff25b183a4d3733801ace5c9d09, store=B 2023-09-29T06:14:42,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:42,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 18784ff25b183a4d3733801ace5c9d09, store=C 2023-09-29T06:14:42,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:42,316 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309298199029a9c854083ba4e3604beb6fe10_18784ff25b183a4d3733801ace5c9d09 is 50, key is test_row_0/A:col10/1695968082302/Put/seqid=0 2023-09-29T06:14:42,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:42,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52114 deadline: 1695968142313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:42,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:42,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968142318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:42,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:42,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968142318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:42,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:42,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52146 deadline: 1695968142319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:42,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:42,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52174 deadline: 1695968142319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:42,329 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309298199029a9c854083ba4e3604beb6fe10_18784ff25b183a4d3733801ace5c9d09 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309298199029a9c854083ba4e3604beb6fe10_18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:14:42,342 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/af6e6836c282475e8c747a7fadca8f86, store: [table=TestAcidGuarantees family=A region=18784ff25b183a4d3733801ace5c9d09] 2023-09-29T06:14:42,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/af6e6836c282475e8c747a7fadca8f86 is 175, key is test_row_0/A:col10/1695968082302/Put/seqid=0 2023-09-29T06:14:42,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:42,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52114 deadline: 1695968142420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:42,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:42,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968142425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:42,430 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2023-09-29T06:14:42,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:42,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. as already flushing 2023-09-29T06:14:42,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:42,431 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(53): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:42,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:42,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:42,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:42,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52146 deadline: 1695968142431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:42,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:42,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968142431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:42,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:42,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52174 deadline: 1695968142431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:42,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2023-09-29T06:14:42,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:42,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. as already flushing 2023-09-29T06:14:42,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:42,584 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(53): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:42,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:42,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:42,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:42,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52114 deadline: 1695968142623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:42,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:42,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968142628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:42,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:42,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52146 deadline: 1695968142634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:42,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:42,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968142635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:42,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:42,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52174 deadline: 1695968142636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:42,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2023-09-29T06:14:42,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:42,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. as already flushing 2023-09-29T06:14:42,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:42,738 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(53): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:42,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:42,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:42,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=107 2023-09-29T06:14:42,767 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/af6e6836c282475e8c747a7fadca8f86 2023-09-29T06:14:42,775 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/c0f5feff7c904ea7b1862ba1ba033ff6 is 50, key is test_row_0/B:col10/1695968082302/Put/seqid=0 2023-09-29T06:14:42,805 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/c0f5feff7c904ea7b1862ba1ba033ff6 2023-09-29T06:14:42,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/1f9b0591292643fca3eafe3be22a1db6 is 50, key is test_row_0/C:col10/1695968082302/Put/seqid=0 2023-09-29T06:14:42,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2023-09-29T06:14:42,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:42,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. as already flushing 2023-09-29T06:14:42,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:42,890 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(53): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:42,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:42,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:42,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:42,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52114 deadline: 1695968142928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:42,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:42,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968142933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:42,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:42,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968142940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:42,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:42,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52146 deadline: 1695968142941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:42,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:42,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52174 deadline: 1695968142947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:43,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2023-09-29T06:14:43,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:43,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. as already flushing 2023-09-29T06:14:43,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:43,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(53): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:43,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:43,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:43,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2023-09-29T06:14:43,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:43,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. as already flushing 2023-09-29T06:14:43,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:43,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(53): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:43,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:43,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:43,237 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/1f9b0591292643fca3eafe3be22a1db6 2023-09-29T06:14:43,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/af6e6836c282475e8c747a7fadca8f86 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/af6e6836c282475e8c747a7fadca8f86 2023-09-29T06:14:43,250 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/af6e6836c282475e8c747a7fadca8f86, entries=200, sequenceid=41, filesize=38.6 K 2023-09-29T06:14:43,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/c0f5feff7c904ea7b1862ba1ba033ff6 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/c0f5feff7c904ea7b1862ba1ba033ff6 2023-09-29T06:14:43,256 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/c0f5feff7c904ea7b1862ba1ba033ff6, entries=150, sequenceid=41, filesize=11.7 K 2023-09-29T06:14:43,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/1f9b0591292643fca3eafe3be22a1db6 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/1f9b0591292643fca3eafe3be22a1db6 2023-09-29T06:14:43,265 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/1f9b0591292643fca3eafe3be22a1db6, entries=150, sequenceid=41, filesize=11.7 K 2023-09-29T06:14:43,265 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 18784ff25b183a4d3733801ace5c9d09 in 961ms, sequenceid=41, compaction requested=false 2023-09-29T06:14:43,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 18784ff25b183a4d3733801ace5c9d09: 2023-09-29T06:14:43,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2023-09-29T06:14:43,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:43,353 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2819): Flushing 18784ff25b183a4d3733801ace5c9d09 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2023-09-29T06:14:43,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 18784ff25b183a4d3733801ace5c9d09, store=A 2023-09-29T06:14:43,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:43,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 18784ff25b183a4d3733801ace5c9d09, store=B 2023-09-29T06:14:43,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:43,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 18784ff25b183a4d3733801ace5c9d09, store=C 2023-09-29T06:14:43,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:43,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929a9c3b6c9c5cc4ea6bd698486e6836180_18784ff25b183a4d3733801ace5c9d09 is 50, key is test_row_0/A:col10/1695968082317/Put/seqid=0 2023-09-29T06:14:43,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:14:43,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. as already flushing 2023-09-29T06:14:43,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:43,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968143466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:43,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:43,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52114 deadline: 1695968143467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:43,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:43,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968143471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:43,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:43,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52146 deadline: 1695968143471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:43,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:43,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52174 deadline: 1695968143476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:43,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:43,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52114 deadline: 1695968143577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:43,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:43,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968143577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:43,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:43,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968143577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:43,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:43,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52146 deadline: 1695968143577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:43,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:43,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52174 deadline: 1695968143583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:43,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=107 2023-09-29T06:14:43,784 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929a9c3b6c9c5cc4ea6bd698486e6836180_18784ff25b183a4d3733801ace5c9d09 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929a9c3b6c9c5cc4ea6bd698486e6836180_18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:14:43,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/2a4cdacace1e40a0b31b097bc59912e1, store: [table=TestAcidGuarantees family=A region=18784ff25b183a4d3733801ace5c9d09] 2023-09-29T06:14:43,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/2a4cdacace1e40a0b31b097bc59912e1 is 175, key is test_row_0/A:col10/1695968082317/Put/seqid=0 2023-09-29T06:14:43,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:43,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52114 deadline: 1695968143783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:43,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:43,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968143784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:43,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:43,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968143785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:43,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:43,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52146 deadline: 1695968143785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:43,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:43,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52174 deadline: 1695968143790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:43,808 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/2a4cdacace1e40a0b31b097bc59912e1 2023-09-29T06:14:43,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/18b333da0f2447339da424488cb9f37b is 50, key is test_row_0/B:col10/1695968082317/Put/seqid=0 2023-09-29T06:14:44,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:44,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968144088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:44,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:44,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52114 deadline: 1695968144088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:44,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:44,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968144092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:44,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:44,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52146 deadline: 1695968144094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:44,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:44,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52174 deadline: 1695968144097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:44,231 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/18b333da0f2447339da424488cb9f37b 2023-09-29T06:14:44,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/63f37e3c1000450fa80c97636db2612e is 50, key is test_row_0/C:col10/1695968082317/Put/seqid=0 2023-09-29T06:14:44,308 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2023-09-29T06:14:44,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:44,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968144591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:44,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:44,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52114 deadline: 1695968144594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:44,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:44,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968144598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:44,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:44,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52146 deadline: 1695968144600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:44,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:44,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52174 deadline: 1695968144603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:44,648 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/63f37e3c1000450fa80c97636db2612e 2023-09-29T06:14:44,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/2a4cdacace1e40a0b31b097bc59912e1 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/2a4cdacace1e40a0b31b097bc59912e1 2023-09-29T06:14:44,657 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/2a4cdacace1e40a0b31b097bc59912e1, entries=150, sequenceid=54, filesize=30.2 K 2023-09-29T06:14:44,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/18b333da0f2447339da424488cb9f37b as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/18b333da0f2447339da424488cb9f37b 2023-09-29T06:14:44,664 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/18b333da0f2447339da424488cb9f37b, entries=150, sequenceid=54, filesize=11.7 K 2023-09-29T06:14:44,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/63f37e3c1000450fa80c97636db2612e as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/63f37e3c1000450fa80c97636db2612e 2023-09-29T06:14:44,670 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/63f37e3c1000450fa80c97636db2612e, entries=150, sequenceid=54, filesize=11.7 K 2023-09-29T06:14:44,671 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3022): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 18784ff25b183a4d3733801ace5c9d09 in 1318ms, sequenceid=54, compaction requested=true 2023-09-29T06:14:44,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2520): Flush status journal for 18784ff25b183a4d3733801ace5c9d09: 2023-09-29T06:14:44,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:44,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=108 2023-09-29T06:14:44,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=108 2023-09-29T06:14:44,679 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=108, resume processing ppid=107 2023-09-29T06:14:44,679 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=108, ppid=107, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0330 sec 2023-09-29T06:14:44,680 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=107, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 3.0550 sec 2023-09-29T06:14:45,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:14:45,599 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 18784ff25b183a4d3733801ace5c9d09 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2023-09-29T06:14:45,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 18784ff25b183a4d3733801ace5c9d09, store=A 2023-09-29T06:14:45,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:45,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 18784ff25b183a4d3733801ace5c9d09, store=B 2023-09-29T06:14:45,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:45,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 18784ff25b183a4d3733801ace5c9d09, store=C 2023-09-29T06:14:45,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:45,605 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309296a381e06005446c1ad02ea27f7fbdf59_18784ff25b183a4d3733801ace5c9d09 is 50, key is test_row_0/A:col10/1695968083469/Put/seqid=0 2023-09-29T06:14:45,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:45,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968145615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:45,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:45,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968145616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:45,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:45,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52146 deadline: 1695968145616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:45,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:45,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:45,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52114 deadline: 1695968145617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:45,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52174 deadline: 1695968145617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:45,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:45,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968145721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:45,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:45,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968145724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:45,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:45,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52146 deadline: 1695968145724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:45,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:45,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52174 deadline: 1695968145724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:45,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:45,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52114 deadline: 1695968145725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:45,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=107 2023-09-29T06:14:45,746 INFO [Thread-2144 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 107 completed 2023-09-29T06:14:45,747 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:14:45,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2023-09-29T06:14:45,750 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:14:45,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=109 2023-09-29T06:14:45,750 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:14:45,750 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:14:45,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=109 2023-09-29T06:14:45,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2023-09-29T06:14:45,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:45,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. as already flushing 2023-09-29T06:14:45,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:45,904 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(53): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:45,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:45,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:45,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:45,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52174 deadline: 1695968145927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:45,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:45,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968145927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:45,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:45,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968145928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:45,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:45,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52146 deadline: 1695968145928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:45,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:45,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52114 deadline: 1695968145928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:46,024 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309296a381e06005446c1ad02ea27f7fbdf59_18784ff25b183a4d3733801ace5c9d09 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309296a381e06005446c1ad02ea27f7fbdf59_18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:14:46,025 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/98953aecc029445abaf1714cc2bea07d, store: [table=TestAcidGuarantees family=A region=18784ff25b183a4d3733801ace5c9d09] 2023-09-29T06:14:46,025 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/98953aecc029445abaf1714cc2bea07d is 175, key is test_row_0/A:col10/1695968083469/Put/seqid=0 2023-09-29T06:14:46,029 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/98953aecc029445abaf1714cc2bea07d 2023-09-29T06:14:46,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/f785dddbc0d64b9e81a45519454313e5 is 50, key is test_row_0/B:col10/1695968083469/Put/seqid=0 2023-09-29T06:14:46,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=109 2023-09-29T06:14:46,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2023-09-29T06:14:46,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:46,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. as already flushing 2023-09-29T06:14:46,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:46,056 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(53): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:46,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:46,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:46,208 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2023-09-29T06:14:46,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:46,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. as already flushing 2023-09-29T06:14:46,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:46,208 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(53): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:46,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:46,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:46,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:46,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52174 deadline: 1695968146231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:46,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:46,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968146231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:46,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:46,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52114 deadline: 1695968146232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:46,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:46,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968146233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:46,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:46,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52146 deadline: 1695968146233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:46,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=109 2023-09-29T06:14:46,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2023-09-29T06:14:46,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:46,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. as already flushing 2023-09-29T06:14:46,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:46,363 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(53): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:46,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:46,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:46,438 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/f785dddbc0d64b9e81a45519454313e5 2023-09-29T06:14:46,446 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/284043b7fae9498bae42c1f08b73c64d is 50, key is test_row_0/C:col10/1695968083469/Put/seqid=0 2023-09-29T06:14:46,454 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/284043b7fae9498bae42c1f08b73c64d 2023-09-29T06:14:46,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/98953aecc029445abaf1714cc2bea07d as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/98953aecc029445abaf1714cc2bea07d 2023-09-29T06:14:46,468 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/98953aecc029445abaf1714cc2bea07d, entries=200, sequenceid=78, filesize=38.6 K 2023-09-29T06:14:46,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/f785dddbc0d64b9e81a45519454313e5 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/f785dddbc0d64b9e81a45519454313e5 2023-09-29T06:14:46,472 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/f785dddbc0d64b9e81a45519454313e5, entries=150, sequenceid=78, filesize=11.7 K 2023-09-29T06:14:46,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/284043b7fae9498bae42c1f08b73c64d as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/284043b7fae9498bae42c1f08b73c64d 2023-09-29T06:14:46,476 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/284043b7fae9498bae42c1f08b73c64d, entries=150, sequenceid=78, filesize=11.7 K 2023-09-29T06:14:46,477 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 18784ff25b183a4d3733801ace5c9d09 in 877ms, sequenceid=78, compaction requested=true 2023-09-29T06:14:46,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 18784ff25b183a4d3733801ace5c9d09: 2023-09-29T06:14:46,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18784ff25b183a4d3733801ace5c9d09:A, priority=-2147483648, current under compaction store size is 1 2023-09-29T06:14:46,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:46,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18784ff25b183a4d3733801ace5c9d09:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:14:46,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:46,477 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-29T06:14:46,477 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-29T06:14:46,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18784ff25b183a4d3733801ace5c9d09:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:14:46,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:14:46,478 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 140940 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-29T06:14:46,478 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 18784ff25b183a4d3733801ace5c9d09/A is initiating minor compaction (all files) 2023-09-29T06:14:46,478 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47936 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-29T06:14:46,478 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 18784ff25b183a4d3733801ace5c9d09/A in TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:46,478 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 18784ff25b183a4d3733801ace5c9d09/B is initiating minor compaction (all files) 2023-09-29T06:14:46,478 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/3f28254d3ffb4eeb807f5265d15371f3, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/af6e6836c282475e8c747a7fadca8f86, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/2a4cdacace1e40a0b31b097bc59912e1, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/98953aecc029445abaf1714cc2bea07d] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp, totalSize=137.6 K 2023-09-29T06:14:46,478 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 18784ff25b183a4d3733801ace5c9d09/B in TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:46,478 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:46,478 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. files: [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/3f28254d3ffb4eeb807f5265d15371f3, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/af6e6836c282475e8c747a7fadca8f86, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/2a4cdacace1e40a0b31b097bc59912e1, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/98953aecc029445abaf1714cc2bea07d] 2023-09-29T06:14:46,478 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/5300f1bc83354dd9ae4085650173502b, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/c0f5feff7c904ea7b1862ba1ba033ff6, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/18b333da0f2447339da424488cb9f37b, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/f785dddbc0d64b9e81a45519454313e5] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp, totalSize=46.8 K 2023-09-29T06:14:46,479 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 3f28254d3ffb4eeb807f5265d15371f3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=18, earliestPutTs=1695968081665 2023-09-29T06:14:46,479 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 5300f1bc83354dd9ae4085650173502b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=18, earliestPutTs=1695968081665 2023-09-29T06:14:46,479 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting af6e6836c282475e8c747a7fadca8f86, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1695968081687 2023-09-29T06:14:46,479 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting c0f5feff7c904ea7b1862ba1ba033ff6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1695968081688 2023-09-29T06:14:46,479 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 2a4cdacace1e40a0b31b097bc59912e1, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1695968082305 2023-09-29T06:14:46,480 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 18b333da0f2447339da424488cb9f37b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1695968082305 2023-09-29T06:14:46,481 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting f785dddbc0d64b9e81a45519454313e5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1695968083466 2023-09-29T06:14:46,481 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 98953aecc029445abaf1714cc2bea07d, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1695968083466 2023-09-29T06:14:46,501 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=18784ff25b183a4d3733801ace5c9d09] 2023-09-29T06:14:46,502 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18784ff25b183a4d3733801ace5c9d09#B#compaction#360 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:14:46,502 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/8486c8e6cecf46bbb0a38dbd766f7742 is 50, key is test_row_0/B:col10/1695968083469/Put/seqid=0 2023-09-29T06:14:46,503 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202309296dc2ac9e99694c67a0993f3c888d56cb_18784ff25b183a4d3733801ace5c9d09 store=[table=TestAcidGuarantees family=A region=18784ff25b183a4d3733801ace5c9d09] 2023-09-29T06:14:46,505 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202309296dc2ac9e99694c67a0993f3c888d56cb_18784ff25b183a4d3733801ace5c9d09, store=[table=TestAcidGuarantees family=A region=18784ff25b183a4d3733801ace5c9d09] 2023-09-29T06:14:46,505 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309296dc2ac9e99694c67a0993f3c888d56cb_18784ff25b183a4d3733801ace5c9d09 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=18784ff25b183a4d3733801ace5c9d09] 2023-09-29T06:14:46,511 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/8486c8e6cecf46bbb0a38dbd766f7742 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/8486c8e6cecf46bbb0a38dbd766f7742 2023-09-29T06:14:46,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2023-09-29T06:14:46,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:46,516 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2819): Flushing 18784ff25b183a4d3733801ace5c9d09 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2023-09-29T06:14:46,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 18784ff25b183a4d3733801ace5c9d09, store=A 2023-09-29T06:14:46,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:46,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 18784ff25b183a4d3733801ace5c9d09, store=B 2023-09-29T06:14:46,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:46,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 18784ff25b183a4d3733801ace5c9d09, store=C 2023-09-29T06:14:46,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:46,517 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18784ff25b183a4d3733801ace5c9d09#A#compaction#361 average throughput is 1.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:14:46,518 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/48246e66a92344cea05a1e41a8d99d8c is 175, key is test_row_0/A:col10/1695968083469/Put/seqid=0 2023-09-29T06:14:46,527 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 18784ff25b183a4d3733801ace5c9d09/B of 18784ff25b183a4d3733801ace5c9d09 into 8486c8e6cecf46bbb0a38dbd766f7742(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:14:46,527 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 18784ff25b183a4d3733801ace5c9d09: 2023-09-29T06:14:46,527 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09., storeName=18784ff25b183a4d3733801ace5c9d09/B, priority=12, startTime=1695968086477; duration=0sec 2023-09-29T06:14:46,527 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:14:46,527 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18784ff25b183a4d3733801ace5c9d09:B 2023-09-29T06:14:46,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309294e7fffac01e04ee6947f444bc21f0af0_18784ff25b183a4d3733801ace5c9d09 is 50, key is test_row_0/A:col10/1695968085616/Put/seqid=0 2023-09-29T06:14:46,527 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-29T06:14:46,532 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47936 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-29T06:14:46,533 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 18784ff25b183a4d3733801ace5c9d09/C is initiating minor compaction (all files) 2023-09-29T06:14:46,533 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 18784ff25b183a4d3733801ace5c9d09/C in TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:46,533 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/1d141bd1a12241b3abbca77fd2290c24, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/1f9b0591292643fca3eafe3be22a1db6, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/63f37e3c1000450fa80c97636db2612e, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/284043b7fae9498bae42c1f08b73c64d] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp, totalSize=46.8 K 2023-09-29T06:14:46,533 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 1d141bd1a12241b3abbca77fd2290c24, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=18, earliestPutTs=1695968081665 2023-09-29T06:14:46,534 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 1f9b0591292643fca3eafe3be22a1db6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1695968081688 2023-09-29T06:14:46,535 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 63f37e3c1000450fa80c97636db2612e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1695968082305 2023-09-29T06:14:46,535 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 284043b7fae9498bae42c1f08b73c64d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1695968083466 2023-09-29T06:14:46,542 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18784ff25b183a4d3733801ace5c9d09#C#compaction#363 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:14:46,543 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/2e67ec3f817d4407b59a53f2fb2187c0 is 50, key is test_row_0/C:col10/1695968083469/Put/seqid=0 2023-09-29T06:14:46,551 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/48246e66a92344cea05a1e41a8d99d8c as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/48246e66a92344cea05a1e41a8d99d8c 2023-09-29T06:14:46,555 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 18784ff25b183a4d3733801ace5c9d09/A of 18784ff25b183a4d3733801ace5c9d09 into 48246e66a92344cea05a1e41a8d99d8c(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:14:46,555 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 18784ff25b183a4d3733801ace5c9d09: 2023-09-29T06:14:46,555 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09., storeName=18784ff25b183a4d3733801ace5c9d09/A, priority=12, startTime=1695968086477; duration=0sec 2023-09-29T06:14:46,555 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:46,555 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18784ff25b183a4d3733801ace5c9d09:A 2023-09-29T06:14:46,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:14:46,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. as already flushing 2023-09-29T06:14:46,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:46,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968146757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:46,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:46,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968146757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:46,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:46,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52146 deadline: 1695968146759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:46,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:46,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52174 deadline: 1695968146762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:46,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:46,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52114 deadline: 1695968146763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:46,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=109 2023-09-29T06:14:46,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:46,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968146866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:46,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:46,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968146866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:46,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:46,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52146 deadline: 1695968146866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:46,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:46,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52174 deadline: 1695968146867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:46,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:46,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52114 deadline: 1695968146868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:46,953 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309294e7fffac01e04ee6947f444bc21f0af0_18784ff25b183a4d3733801ace5c9d09 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309294e7fffac01e04ee6947f444bc21f0af0_18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:14:46,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/aa06083636b14f98a45965fd9c80d099, store: [table=TestAcidGuarantees family=A region=18784ff25b183a4d3733801ace5c9d09] 2023-09-29T06:14:46,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/aa06083636b14f98a45965fd9c80d099 is 175, key is test_row_0/A:col10/1695968085616/Put/seqid=0 2023-09-29T06:14:46,956 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/2e67ec3f817d4407b59a53f2fb2187c0 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/2e67ec3f817d4407b59a53f2fb2187c0 2023-09-29T06:14:46,959 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=90, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/aa06083636b14f98a45965fd9c80d099 2023-09-29T06:14:46,962 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 18784ff25b183a4d3733801ace5c9d09/C of 18784ff25b183a4d3733801ace5c9d09 into 2e67ec3f817d4407b59a53f2fb2187c0(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:14:46,962 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 18784ff25b183a4d3733801ace5c9d09: 2023-09-29T06:14:46,962 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09., storeName=18784ff25b183a4d3733801ace5c9d09/C, priority=12, startTime=1695968086477; duration=0sec 2023-09-29T06:14:46,962 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:46,962 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18784ff25b183a4d3733801ace5c9d09:C 2023-09-29T06:14:46,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/b01705fc4c7543f1ba76de60023bc591 is 50, key is test_row_0/B:col10/1695968085616/Put/seqid=0 2023-09-29T06:14:47,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:47,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968147069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:47,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:47,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968147070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:47,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:47,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52146 deadline: 1695968147070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:47,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:47,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52174 deadline: 1695968147071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:47,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:47,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52114 deadline: 1695968147072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:47,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:47,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52114 deadline: 1695968147378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:47,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:47,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968147378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:47,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:47,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52146 deadline: 1695968147378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:47,379 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/b01705fc4c7543f1ba76de60023bc591 2023-09-29T06:14:47,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:47,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52174 deadline: 1695968147378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:47,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:47,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968147382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:47,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/96e93103a4854ba59af31f6008e44357 is 50, key is test_row_0/C:col10/1695968085616/Put/seqid=0 2023-09-29T06:14:47,796 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/96e93103a4854ba59af31f6008e44357 2023-09-29T06:14:47,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/aa06083636b14f98a45965fd9c80d099 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/aa06083636b14f98a45965fd9c80d099 2023-09-29T06:14:47,803 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/aa06083636b14f98a45965fd9c80d099, entries=150, sequenceid=90, filesize=30.2 K 2023-09-29T06:14:47,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/b01705fc4c7543f1ba76de60023bc591 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/b01705fc4c7543f1ba76de60023bc591 2023-09-29T06:14:47,807 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/b01705fc4c7543f1ba76de60023bc591, entries=150, sequenceid=90, filesize=11.7 K 2023-09-29T06:14:47,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/96e93103a4854ba59af31f6008e44357 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/96e93103a4854ba59af31f6008e44357 2023-09-29T06:14:47,811 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/96e93103a4854ba59af31f6008e44357, entries=150, sequenceid=90, filesize=11.7 K 2023-09-29T06:14:47,812 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3022): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 18784ff25b183a4d3733801ace5c9d09 in 1295ms, sequenceid=90, compaction requested=false 2023-09-29T06:14:47,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2520): Flush status journal for 18784ff25b183a4d3733801ace5c9d09: 2023-09-29T06:14:47,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:47,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=110 2023-09-29T06:14:47,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=110 2023-09-29T06:14:47,814 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=110, resume processing ppid=109 2023-09-29T06:14:47,814 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=110, ppid=109, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0630 sec 2023-09-29T06:14:47,815 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=109, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 2.0670 sec 2023-09-29T06:14:47,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=109 2023-09-29T06:14:47,854 INFO [Thread-2144 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 109 completed 2023-09-29T06:14:47,855 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:14:47,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2023-09-29T06:14:47,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=111 2023-09-29T06:14:47,857 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:14:47,858 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:14:47,858 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:14:47,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:14:47,881 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 18784ff25b183a4d3733801ace5c9d09 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2023-09-29T06:14:47,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 18784ff25b183a4d3733801ace5c9d09, store=A 2023-09-29T06:14:47,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:47,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 18784ff25b183a4d3733801ace5c9d09, store=B 2023-09-29T06:14:47,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:47,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 18784ff25b183a4d3733801ace5c9d09, store=C 2023-09-29T06:14:47,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:47,887 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929f7bef947d2eb4328ac8a8da82f9b1ec8_18784ff25b183a4d3733801ace5c9d09 is 50, key is test_row_0/A:col10/1695968086755/Put/seqid=0 2023-09-29T06:14:47,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:47,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968147888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:47,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:47,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968147888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:47,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:47,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52146 deadline: 1695968147891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:47,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:47,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52174 deadline: 1695968147892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:47,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:47,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52114 deadline: 1695968147893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:47,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=111 2023-09-29T06:14:47,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:47,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968147994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:47,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:47,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968147994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:47,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:47,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52146 deadline: 1695968147994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:47,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:47,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52174 deadline: 1695968147996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:48,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:48,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52114 deadline: 1695968147999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:48,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2023-09-29T06:14:48,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:48,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. as already flushing 2023-09-29T06:14:48,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:48,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(53): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:48,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:48,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:48,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=111 2023-09-29T06:14:48,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2023-09-29T06:14:48,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:48,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. as already flushing 2023-09-29T06:14:48,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:48,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(53): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:48,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:48,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:48,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:48,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968148197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:48,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:48,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968148197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:48,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:48,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52146 deadline: 1695968148198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:48,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:48,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52174 deadline: 1695968148199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:48,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:48,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52114 deadline: 1695968148202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:48,295 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929f7bef947d2eb4328ac8a8da82f9b1ec8_18784ff25b183a4d3733801ace5c9d09 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929f7bef947d2eb4328ac8a8da82f9b1ec8_18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:14:48,295 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/cf3aa8ca473b4285b872a1478f8135d0, store: [table=TestAcidGuarantees family=A region=18784ff25b183a4d3733801ace5c9d09] 2023-09-29T06:14:48,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/cf3aa8ca473b4285b872a1478f8135d0 is 175, key is test_row_0/A:col10/1695968086755/Put/seqid=0 2023-09-29T06:14:48,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2023-09-29T06:14:48,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:48,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. as already flushing 2023-09-29T06:14:48,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:48,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(53): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:48,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:48,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:48,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=111 2023-09-29T06:14:48,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2023-09-29T06:14:48,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:48,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. as already flushing 2023-09-29T06:14:48,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:48,468 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(53): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:48,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:48,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:48,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:48,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968148500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:48,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:48,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968148500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:48,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:48,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52146 deadline: 1695968148501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:48,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:48,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52114 deadline: 1695968148504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:48,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:48,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52174 deadline: 1695968148504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:48,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2023-09-29T06:14:48,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:48,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. as already flushing 2023-09-29T06:14:48,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:48,620 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(53): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:48,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:48,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:48,700 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/cf3aa8ca473b4285b872a1478f8135d0 2023-09-29T06:14:48,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/b10dbcadcd1b44e0ba55fc38e4f123e5 is 50, key is test_row_0/B:col10/1695968086755/Put/seqid=0 2023-09-29T06:14:48,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2023-09-29T06:14:48,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:48,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. as already flushing 2023-09-29T06:14:48,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:48,773 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(53): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:48,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:48,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:48,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2023-09-29T06:14:48,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:48,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. as already flushing 2023-09-29T06:14:48,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:48,926 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(53): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:48,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:48,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:48,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=111 2023-09-29T06:14:49,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:49,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52146 deadline: 1695968149004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:49,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:49,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968149006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:49,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:49,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968149008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:49,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:49,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52114 deadline: 1695968149010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:49,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:49,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52174 deadline: 1695968149011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:49,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2023-09-29T06:14:49,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:49,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. as already flushing 2023-09-29T06:14:49,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:49,078 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(53): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:49,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:49,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:49,110 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/b10dbcadcd1b44e0ba55fc38e4f123e5 2023-09-29T06:14:49,116 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/92ba6f4e97ac4325918303ece8909753 is 50, key is test_row_0/C:col10/1695968086755/Put/seqid=0 2023-09-29T06:14:49,230 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2023-09-29T06:14:49,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:49,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. as already flushing 2023-09-29T06:14:49,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:49,231 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(53): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:49,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:49,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:49,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2023-09-29T06:14:49,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:49,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. as already flushing 2023-09-29T06:14:49,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:49,383 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(53): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:49,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:49,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:49,520 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/92ba6f4e97ac4325918303ece8909753 2023-09-29T06:14:49,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/cf3aa8ca473b4285b872a1478f8135d0 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/cf3aa8ca473b4285b872a1478f8135d0 2023-09-29T06:14:49,528 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/cf3aa8ca473b4285b872a1478f8135d0, entries=200, sequenceid=118, filesize=38.6 K 2023-09-29T06:14:49,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/b10dbcadcd1b44e0ba55fc38e4f123e5 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/b10dbcadcd1b44e0ba55fc38e4f123e5 2023-09-29T06:14:49,532 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/b10dbcadcd1b44e0ba55fc38e4f123e5, entries=150, sequenceid=118, filesize=11.7 K 2023-09-29T06:14:49,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/92ba6f4e97ac4325918303ece8909753 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/92ba6f4e97ac4325918303ece8909753 2023-09-29T06:14:49,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2023-09-29T06:14:49,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:49,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. as already flushing 2023-09-29T06:14:49,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:49,536 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(53): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:49,536 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/92ba6f4e97ac4325918303ece8909753, entries=150, sequenceid=118, filesize=11.7 K 2023-09-29T06:14:49,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:49,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:49,538 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 18784ff25b183a4d3733801ace5c9d09 in 1657ms, sequenceid=118, compaction requested=true 2023-09-29T06:14:49,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 18784ff25b183a4d3733801ace5c9d09: 2023-09-29T06:14:49,538 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:14:49,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18784ff25b183a4d3733801ace5c9d09:A, priority=-2147483648, current under compaction store size is 1 2023-09-29T06:14:49,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:49,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18784ff25b183a4d3733801ace5c9d09:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:14:49,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:49,538 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:14:49,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18784ff25b183a4d3733801ace5c9d09:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:14:49,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:14:49,540 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101546 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:14:49,540 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36090 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:14:49,540 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 18784ff25b183a4d3733801ace5c9d09/A is initiating minor compaction (all files) 2023-09-29T06:14:49,540 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 18784ff25b183a4d3733801ace5c9d09/B is initiating minor compaction (all files) 2023-09-29T06:14:49,540 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 18784ff25b183a4d3733801ace5c9d09/A in TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:49,540 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 18784ff25b183a4d3733801ace5c9d09/B in TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:49,540 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/48246e66a92344cea05a1e41a8d99d8c, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/aa06083636b14f98a45965fd9c80d099, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/cf3aa8ca473b4285b872a1478f8135d0] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp, totalSize=99.2 K 2023-09-29T06:14:49,540 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/8486c8e6cecf46bbb0a38dbd766f7742, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/b01705fc4c7543f1ba76de60023bc591, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/b10dbcadcd1b44e0ba55fc38e4f123e5] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp, totalSize=35.2 K 2023-09-29T06:14:49,540 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:49,540 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. files: [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/48246e66a92344cea05a1e41a8d99d8c, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/aa06083636b14f98a45965fd9c80d099, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/cf3aa8ca473b4285b872a1478f8135d0] 2023-09-29T06:14:49,541 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 8486c8e6cecf46bbb0a38dbd766f7742, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1695968083466 2023-09-29T06:14:49,541 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 48246e66a92344cea05a1e41a8d99d8c, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1695968083466 2023-09-29T06:14:49,541 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting b01705fc4c7543f1ba76de60023bc591, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1695968085614 2023-09-29T06:14:49,541 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting aa06083636b14f98a45965fd9c80d099, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1695968085614 2023-09-29T06:14:49,541 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting b10dbcadcd1b44e0ba55fc38e4f123e5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1695968086755 2023-09-29T06:14:49,542 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting cf3aa8ca473b4285b872a1478f8135d0, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1695968086755 2023-09-29T06:14:49,547 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=18784ff25b183a4d3733801ace5c9d09] 2023-09-29T06:14:49,551 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20230929b8e9da44e321464d9bf0f3062d1c9e1f_18784ff25b183a4d3733801ace5c9d09 store=[table=TestAcidGuarantees family=A region=18784ff25b183a4d3733801ace5c9d09] 2023-09-29T06:14:49,552 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18784ff25b183a4d3733801ace5c9d09#B#compaction#370 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:14:49,553 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/0c0fa39865e24dc5850f6a17de895131 is 50, key is test_row_0/B:col10/1695968086755/Put/seqid=0 2023-09-29T06:14:49,558 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20230929b8e9da44e321464d9bf0f3062d1c9e1f_18784ff25b183a4d3733801ace5c9d09, store=[table=TestAcidGuarantees family=A region=18784ff25b183a4d3733801ace5c9d09] 2023-09-29T06:14:49,558 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929b8e9da44e321464d9bf0f3062d1c9e1f_18784ff25b183a4d3733801ace5c9d09 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=18784ff25b183a4d3733801ace5c9d09] 2023-09-29T06:14:49,574 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/0c0fa39865e24dc5850f6a17de895131 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/0c0fa39865e24dc5850f6a17de895131 2023-09-29T06:14:49,587 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18784ff25b183a4d3733801ace5c9d09#A#compaction#369 average throughput is 0.61 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:14:49,587 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/867c273436c94ea1af624e3530032498 is 175, key is test_row_0/A:col10/1695968086755/Put/seqid=0 2023-09-29T06:14:49,591 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 18784ff25b183a4d3733801ace5c9d09/B of 18784ff25b183a4d3733801ace5c9d09 into 0c0fa39865e24dc5850f6a17de895131(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:14:49,591 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 18784ff25b183a4d3733801ace5c9d09: 2023-09-29T06:14:49,591 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09., storeName=18784ff25b183a4d3733801ace5c9d09/B, priority=13, startTime=1695968089538; duration=0sec 2023-09-29T06:14:49,591 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:14:49,591 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18784ff25b183a4d3733801ace5c9d09:B 2023-09-29T06:14:49,591 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:14:49,592 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36090 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:14:49,592 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 18784ff25b183a4d3733801ace5c9d09/C is initiating minor compaction (all files) 2023-09-29T06:14:49,592 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 18784ff25b183a4d3733801ace5c9d09/C in TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:49,592 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/2e67ec3f817d4407b59a53f2fb2187c0, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/96e93103a4854ba59af31f6008e44357, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/92ba6f4e97ac4325918303ece8909753] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp, totalSize=35.2 K 2023-09-29T06:14:49,592 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 2e67ec3f817d4407b59a53f2fb2187c0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1695968083466 2023-09-29T06:14:49,593 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 96e93103a4854ba59af31f6008e44357, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1695968085614 2023-09-29T06:14:49,593 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 92ba6f4e97ac4325918303ece8909753, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1695968086755 2023-09-29T06:14:49,619 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18784ff25b183a4d3733801ace5c9d09#C#compaction#371 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:14:49,620 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/83f70e818439417bbd5dff786adc0235 is 50, key is test_row_0/C:col10/1695968086755/Put/seqid=0 2023-09-29T06:14:49,623 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/867c273436c94ea1af624e3530032498 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/867c273436c94ea1af624e3530032498 2023-09-29T06:14:49,640 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 18784ff25b183a4d3733801ace5c9d09/A of 18784ff25b183a4d3733801ace5c9d09 into 867c273436c94ea1af624e3530032498(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:14:49,640 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 18784ff25b183a4d3733801ace5c9d09: 2023-09-29T06:14:49,640 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09., storeName=18784ff25b183a4d3733801ace5c9d09/A, priority=13, startTime=1695968089538; duration=0sec 2023-09-29T06:14:49,640 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:49,640 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18784ff25b183a4d3733801ace5c9d09:A 2023-09-29T06:14:49,654 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/83f70e818439417bbd5dff786adc0235 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/83f70e818439417bbd5dff786adc0235 2023-09-29T06:14:49,658 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 18784ff25b183a4d3733801ace5c9d09/C of 18784ff25b183a4d3733801ace5c9d09 into 83f70e818439417bbd5dff786adc0235(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:14:49,658 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 18784ff25b183a4d3733801ace5c9d09: 2023-09-29T06:14:49,658 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09., storeName=18784ff25b183a4d3733801ace5c9d09/C, priority=13, startTime=1695968089538; duration=0sec 2023-09-29T06:14:49,658 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:49,658 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18784ff25b183a4d3733801ace5c9d09:C 2023-09-29T06:14:49,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2023-09-29T06:14:49,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:49,689 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2819): Flushing 18784ff25b183a4d3733801ace5c9d09 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2023-09-29T06:14:49,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 18784ff25b183a4d3733801ace5c9d09, store=A 2023-09-29T06:14:49,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:49,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 18784ff25b183a4d3733801ace5c9d09, store=B 2023-09-29T06:14:49,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:49,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 18784ff25b183a4d3733801ace5c9d09, store=C 2023-09-29T06:14:49,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:49,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023092915d77c8d351945f9aa91402b7827fda5_18784ff25b183a4d3733801ace5c9d09 is 50, key is test_row_0/A:col10/1695968087892/Put/seqid=0 2023-09-29T06:14:49,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=111 2023-09-29T06:14:50,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. as already flushing 2023-09-29T06:14:50,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:14:50,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:50,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968150059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:50,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:50,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52114 deadline: 1695968150060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:50,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:50,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52174 deadline: 1695968150061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:50,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:50,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52146 deadline: 1695968150061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:50,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:50,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968150063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:50,115 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023092915d77c8d351945f9aa91402b7827fda5_18784ff25b183a4d3733801ace5c9d09 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023092915d77c8d351945f9aa91402b7827fda5_18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:14:50,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/1252f9dcf8f24e2b9714a79de5eeefac, store: [table=TestAcidGuarantees family=A region=18784ff25b183a4d3733801ace5c9d09] 2023-09-29T06:14:50,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/1252f9dcf8f24e2b9714a79de5eeefac is 175, key is test_row_0/A:col10/1695968087892/Put/seqid=0 2023-09-29T06:14:50,121 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/1252f9dcf8f24e2b9714a79de5eeefac 2023-09-29T06:14:50,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/ccbf73ca14994422aa38aca82d258a93 is 50, key is test_row_0/B:col10/1695968087892/Put/seqid=0 2023-09-29T06:14:50,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:50,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968150164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:50,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:50,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52114 deadline: 1695968150164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:50,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:50,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52174 deadline: 1695968150167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:50,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:50,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52146 deadline: 1695968150168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:50,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:50,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968150168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:50,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:50,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968150368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:50,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:50,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52114 deadline: 1695968150368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:50,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:50,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52174 deadline: 1695968150370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:50,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:50,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968150371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:50,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:50,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52146 deadline: 1695968150372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:50,531 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/ccbf73ca14994422aa38aca82d258a93 2023-09-29T06:14:50,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/5ba41f4a26a54cdd9c9178eeb2624ee3 is 50, key is test_row_0/C:col10/1695968087892/Put/seqid=0 2023-09-29T06:14:50,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:50,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968150671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:50,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:50,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52114 deadline: 1695968150673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:50,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:50,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52174 deadline: 1695968150674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:50,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:50,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968150676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:50,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:50,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52146 deadline: 1695968150676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:50,941 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/5ba41f4a26a54cdd9c9178eeb2624ee3 2023-09-29T06:14:50,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/1252f9dcf8f24e2b9714a79de5eeefac as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/1252f9dcf8f24e2b9714a79de5eeefac 2023-09-29T06:14:50,957 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/1252f9dcf8f24e2b9714a79de5eeefac, entries=100, sequenceid=132, filesize=21.8 K 2023-09-29T06:14:50,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/ccbf73ca14994422aa38aca82d258a93 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/ccbf73ca14994422aa38aca82d258a93 2023-09-29T06:14:50,962 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/ccbf73ca14994422aa38aca82d258a93, entries=100, sequenceid=132, filesize=9.4 K 2023-09-29T06:14:50,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/5ba41f4a26a54cdd9c9178eeb2624ee3 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/5ba41f4a26a54cdd9c9178eeb2624ee3 2023-09-29T06:14:50,968 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/5ba41f4a26a54cdd9c9178eeb2624ee3, entries=100, sequenceid=132, filesize=9.4 K 2023-09-29T06:14:50,968 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3022): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 18784ff25b183a4d3733801ace5c9d09 in 1280ms, sequenceid=132, compaction requested=false 2023-09-29T06:14:50,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2520): Flush status journal for 18784ff25b183a4d3733801ace5c9d09: 2023-09-29T06:14:50,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:50,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=112 2023-09-29T06:14:50,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=112 2023-09-29T06:14:50,971 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=112, resume processing ppid=111 2023-09-29T06:14:50,971 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=112, ppid=111, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1110 sec 2023-09-29T06:14:50,972 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=111, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 3.1160 sec 2023-09-29T06:14:51,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:14:51,177 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 18784ff25b183a4d3733801ace5c9d09 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2023-09-29T06:14:51,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 18784ff25b183a4d3733801ace5c9d09, store=A 2023-09-29T06:14:51,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:51,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 18784ff25b183a4d3733801ace5c9d09, store=B 2023-09-29T06:14:51,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:51,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 18784ff25b183a4d3733801ace5c9d09, store=C 2023-09-29T06:14:51,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:51,183 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929f20836bc9dcf43fb8675155f7a5867c2_18784ff25b183a4d3733801ace5c9d09 is 50, key is test_row_0/A:col10/1695968090060/Put/seqid=0 2023-09-29T06:14:51,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:51,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52174 deadline: 1695968151185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:51,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:51,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968151187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:51,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:51,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52114 deadline: 1695968151188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:51,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:51,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968151188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:51,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:51,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52146 deadline: 1695968151189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:51,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:51,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52174 deadline: 1695968151290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:51,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:51,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968151294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:51,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:51,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52114 deadline: 1695968151294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:51,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:51,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968151295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:51,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:51,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52146 deadline: 1695968151295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:51,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:51,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52174 deadline: 1695968151496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:51,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:51,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968151496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:51,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:51,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52114 deadline: 1695968151498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:51,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:51,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968151500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:51,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:51,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52146 deadline: 1695968151501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:51,592 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929f20836bc9dcf43fb8675155f7a5867c2_18784ff25b183a4d3733801ace5c9d09 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929f20836bc9dcf43fb8675155f7a5867c2_18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:14:51,593 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/12148e3f59fb44638107056a104117e5, store: [table=TestAcidGuarantees family=A region=18784ff25b183a4d3733801ace5c9d09] 2023-09-29T06:14:51,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/12148e3f59fb44638107056a104117e5 is 175, key is test_row_0/A:col10/1695968090060/Put/seqid=0 2023-09-29T06:14:51,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:51,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52174 deadline: 1695968151799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:51,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:51,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:51,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968151804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:51,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52114 deadline: 1695968151805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:51,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:51,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968151807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:51,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:51,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52146 deadline: 1695968151808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:51,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=111 2023-09-29T06:14:51,962 INFO [Thread-2144 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 111 completed 2023-09-29T06:14:51,964 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:14:51,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2023-09-29T06:14:51,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=113 2023-09-29T06:14:51,966 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:14:51,967 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:14:51,967 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:14:51,998 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=158, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/12148e3f59fb44638107056a104117e5 2023-09-29T06:14:52,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/96c400958bf2429196354e5a2c8db7da is 50, key is test_row_0/B:col10/1695968090060/Put/seqid=0 2023-09-29T06:14:52,015 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/96c400958bf2429196354e5a2c8db7da 2023-09-29T06:14:52,020 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/d61b5d8ce4df43f2996808a49ab091af is 50, key is test_row_0/C:col10/1695968090060/Put/seqid=0 2023-09-29T06:14:52,029 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/d61b5d8ce4df43f2996808a49ab091af 2023-09-29T06:14:52,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/12148e3f59fb44638107056a104117e5 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/12148e3f59fb44638107056a104117e5 2023-09-29T06:14:52,036 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/12148e3f59fb44638107056a104117e5, entries=150, sequenceid=158, filesize=30.4 K 2023-09-29T06:14:52,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/96c400958bf2429196354e5a2c8db7da as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/96c400958bf2429196354e5a2c8db7da 2023-09-29T06:14:52,041 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/96c400958bf2429196354e5a2c8db7da, entries=150, sequenceid=158, filesize=11.8 K 2023-09-29T06:14:52,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/d61b5d8ce4df43f2996808a49ab091af as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/d61b5d8ce4df43f2996808a49ab091af 2023-09-29T06:14:52,045 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/d61b5d8ce4df43f2996808a49ab091af, entries=150, sequenceid=158, filesize=11.8 K 2023-09-29T06:14:52,046 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 18784ff25b183a4d3733801ace5c9d09 in 869ms, sequenceid=158, compaction requested=true 2023-09-29T06:14:52,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 18784ff25b183a4d3733801ace5c9d09: 2023-09-29T06:14:52,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18784ff25b183a4d3733801ace5c9d09:A, priority=-2147483648, current under compaction store size is 1 2023-09-29T06:14:52,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:52,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18784ff25b183a4d3733801ace5c9d09:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:14:52,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:52,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18784ff25b183a4d3733801ace5c9d09:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:14:52,046 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:14:52,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:14:52,046 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:14:52,047 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 84610 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:14:52,047 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 18784ff25b183a4d3733801ace5c9d09/A is initiating minor compaction (all files) 2023-09-29T06:14:52,047 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 18784ff25b183a4d3733801ace5c9d09/A in TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:52,047 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/867c273436c94ea1af624e3530032498, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/1252f9dcf8f24e2b9714a79de5eeefac, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/12148e3f59fb44638107056a104117e5] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp, totalSize=82.6 K 2023-09-29T06:14:52,047 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:52,047 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. files: [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/867c273436c94ea1af624e3530032498, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/1252f9dcf8f24e2b9714a79de5eeefac, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/12148e3f59fb44638107056a104117e5] 2023-09-29T06:14:52,048 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33998 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:14:52,048 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 18784ff25b183a4d3733801ace5c9d09/B is initiating minor compaction (all files) 2023-09-29T06:14:52,048 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 18784ff25b183a4d3733801ace5c9d09/B in TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:52,048 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 867c273436c94ea1af624e3530032498, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1695968086755 2023-09-29T06:14:52,048 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/0c0fa39865e24dc5850f6a17de895131, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/ccbf73ca14994422aa38aca82d258a93, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/96c400958bf2429196354e5a2c8db7da] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp, totalSize=33.2 K 2023-09-29T06:14:52,049 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 1252f9dcf8f24e2b9714a79de5eeefac, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1695968087890 2023-09-29T06:14:52,049 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 0c0fa39865e24dc5850f6a17de895131, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1695968086755 2023-09-29T06:14:52,050 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 12148e3f59fb44638107056a104117e5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1695968090060 2023-09-29T06:14:52,050 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting ccbf73ca14994422aa38aca82d258a93, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1695968087890 2023-09-29T06:14:52,050 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 96c400958bf2429196354e5a2c8db7da, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1695968090060 2023-09-29T06:14:52,056 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=18784ff25b183a4d3733801ace5c9d09] 2023-09-29T06:14:52,058 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18784ff25b183a4d3733801ace5c9d09#B#compaction#379 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:14:52,058 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/29bc70f1d05e4dc39efc57e1f9afd723 is 50, key is test_row_0/B:col10/1695968090060/Put/seqid=0 2023-09-29T06:14:52,061 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202309294f914b6978e74bbdb1a199cba71904bf_18784ff25b183a4d3733801ace5c9d09 store=[table=TestAcidGuarantees family=A region=18784ff25b183a4d3733801ace5c9d09] 2023-09-29T06:14:52,062 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202309294f914b6978e74bbdb1a199cba71904bf_18784ff25b183a4d3733801ace5c9d09, store=[table=TestAcidGuarantees family=A region=18784ff25b183a4d3733801ace5c9d09] 2023-09-29T06:14:52,062 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309294f914b6978e74bbdb1a199cba71904bf_18784ff25b183a4d3733801ace5c9d09 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=18784ff25b183a4d3733801ace5c9d09] 2023-09-29T06:14:52,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=113 2023-09-29T06:14:52,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2023-09-29T06:14:52,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:52,122 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2819): Flushing 18784ff25b183a4d3733801ace5c9d09 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2023-09-29T06:14:52,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 18784ff25b183a4d3733801ace5c9d09, store=A 2023-09-29T06:14:52,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:52,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 18784ff25b183a4d3733801ace5c9d09, store=B 2023-09-29T06:14:52,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:52,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 18784ff25b183a4d3733801ace5c9d09, store=C 2023-09-29T06:14:52,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:52,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929526ce95b22d04967959e6636526e127c_18784ff25b183a4d3733801ace5c9d09 is 50, key is test_row_0/A:col10/1695968091186/Put/seqid=0 2023-09-29T06:14:52,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=113 2023-09-29T06:14:52,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:14:52,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. as already flushing 2023-09-29T06:14:52,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:52,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968152331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:52,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:52,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52174 deadline: 1695968152332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:52,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:52,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52146 deadline: 1695968152332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:52,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:52,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968152333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:52,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:52,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52114 deadline: 1695968152333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:52,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:52,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968152436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:52,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:52,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52174 deadline: 1695968152437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:52,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:52,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52146 deadline: 1695968152437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:52,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:52,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968152439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:52,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:52,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52114 deadline: 1695968152439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:52,468 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18784ff25b183a4d3733801ace5c9d09#A#compaction#378 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:14:52,469 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/92d35f6e6a37480d812984fd295b38c7 is 175, key is test_row_0/A:col10/1695968090060/Put/seqid=0 2023-09-29T06:14:52,470 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/29bc70f1d05e4dc39efc57e1f9afd723 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/29bc70f1d05e4dc39efc57e1f9afd723 2023-09-29T06:14:52,482 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 18784ff25b183a4d3733801ace5c9d09/B of 18784ff25b183a4d3733801ace5c9d09 into 29bc70f1d05e4dc39efc57e1f9afd723(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:14:52,482 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 18784ff25b183a4d3733801ace5c9d09: 2023-09-29T06:14:52,482 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09., storeName=18784ff25b183a4d3733801ace5c9d09/B, priority=13, startTime=1695968092046; duration=0sec 2023-09-29T06:14:52,482 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:14:52,482 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18784ff25b183a4d3733801ace5c9d09:B 2023-09-29T06:14:52,483 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:14:52,484 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33998 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:14:52,484 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 18784ff25b183a4d3733801ace5c9d09/C is initiating minor compaction (all files) 2023-09-29T06:14:52,484 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 18784ff25b183a4d3733801ace5c9d09/C in TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:52,484 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/83f70e818439417bbd5dff786adc0235, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/5ba41f4a26a54cdd9c9178eeb2624ee3, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/d61b5d8ce4df43f2996808a49ab091af] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp, totalSize=33.2 K 2023-09-29T06:14:52,485 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 83f70e818439417bbd5dff786adc0235, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1695968086755 2023-09-29T06:14:52,485 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 5ba41f4a26a54cdd9c9178eeb2624ee3, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1695968087890 2023-09-29T06:14:52,485 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting d61b5d8ce4df43f2996808a49ab091af, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1695968090060 2023-09-29T06:14:52,491 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18784ff25b183a4d3733801ace5c9d09#C#compaction#381 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:14:52,492 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/53df647d2c624fdc9b487f44b43d4d02 is 50, key is test_row_0/C:col10/1695968090060/Put/seqid=0 2023-09-29T06:14:52,534 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929526ce95b22d04967959e6636526e127c_18784ff25b183a4d3733801ace5c9d09 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929526ce95b22d04967959e6636526e127c_18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:14:52,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/82b85c2897da4d5e814473b2ae66707f, store: [table=TestAcidGuarantees family=A region=18784ff25b183a4d3733801ace5c9d09] 2023-09-29T06:14:52,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/82b85c2897da4d5e814473b2ae66707f is 175, key is test_row_0/A:col10/1695968091186/Put/seqid=0 2023-09-29T06:14:52,541 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=168, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/82b85c2897da4d5e814473b2ae66707f 2023-09-29T06:14:52,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/8ff24e9e4b294023b0bf89da2db935a8 is 50, key is test_row_0/B:col10/1695968091186/Put/seqid=0 2023-09-29T06:14:52,559 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/8ff24e9e4b294023b0bf89da2db935a8 2023-09-29T06:14:52,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/18554b3c86bf428b9dc43f5b170d1fb3 is 50, key is test_row_0/C:col10/1695968091186/Put/seqid=0 2023-09-29T06:14:52,568 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/18554b3c86bf428b9dc43f5b170d1fb3 2023-09-29T06:14:52,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=113 2023-09-29T06:14:52,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/82b85c2897da4d5e814473b2ae66707f as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/82b85c2897da4d5e814473b2ae66707f 2023-09-29T06:14:52,575 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/82b85c2897da4d5e814473b2ae66707f, entries=150, sequenceid=168, filesize=30.4 K 2023-09-29T06:14:52,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/8ff24e9e4b294023b0bf89da2db935a8 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/8ff24e9e4b294023b0bf89da2db935a8 2023-09-29T06:14:52,579 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/8ff24e9e4b294023b0bf89da2db935a8, entries=150, sequenceid=168, filesize=11.8 K 2023-09-29T06:14:52,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/18554b3c86bf428b9dc43f5b170d1fb3 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/18554b3c86bf428b9dc43f5b170d1fb3 2023-09-29T06:14:52,583 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/18554b3c86bf428b9dc43f5b170d1fb3, entries=150, sequenceid=168, filesize=11.8 K 2023-09-29T06:14:52,584 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3022): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 18784ff25b183a4d3733801ace5c9d09 in 462ms, sequenceid=168, compaction requested=false 2023-09-29T06:14:52,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2520): Flush status journal for 18784ff25b183a4d3733801ace5c9d09: 2023-09-29T06:14:52,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:52,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=114 2023-09-29T06:14:52,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=114 2023-09-29T06:14:52,588 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=114, resume processing ppid=113 2023-09-29T06:14:52,588 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=114, ppid=113, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 619 msec 2023-09-29T06:14:52,589 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=113, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 624 msec 2023-09-29T06:14:52,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:14:52,641 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 18784ff25b183a4d3733801ace5c9d09 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2023-09-29T06:14:52,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 18784ff25b183a4d3733801ace5c9d09, store=A 2023-09-29T06:14:52,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:52,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 18784ff25b183a4d3733801ace5c9d09, store=B 2023-09-29T06:14:52,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:52,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 18784ff25b183a4d3733801ace5c9d09, store=C 2023-09-29T06:14:52,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:52,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929b33547c3dac34ae8851cf1220ada2d4f_18784ff25b183a4d3733801ace5c9d09 is 50, key is test_row_0/A:col10/1695968092332/Put/seqid=0 2023-09-29T06:14:52,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:52,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52174 deadline: 1695968152648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:52,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:52,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52146 deadline: 1695968152649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:52,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:52,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968152649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:52,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:52,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52114 deadline: 1695968152650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:52,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:52,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968152655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:52,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:52,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52174 deadline: 1695968152756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:52,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:52,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52146 deadline: 1695968152757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:52,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:52,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968152757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:52,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:52,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52114 deadline: 1695968152757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:52,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:52,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968152761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:52,878 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/92d35f6e6a37480d812984fd295b38c7 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/92d35f6e6a37480d812984fd295b38c7 2023-09-29T06:14:52,881 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 18784ff25b183a4d3733801ace5c9d09/A of 18784ff25b183a4d3733801ace5c9d09 into 92d35f6e6a37480d812984fd295b38c7(size=30.7 K), total size for store is 61.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:14:52,881 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 18784ff25b183a4d3733801ace5c9d09: 2023-09-29T06:14:52,881 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09., storeName=18784ff25b183a4d3733801ace5c9d09/A, priority=13, startTime=1695968092046; duration=0sec 2023-09-29T06:14:52,882 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:52,882 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18784ff25b183a4d3733801ace5c9d09:A 2023-09-29T06:14:52,904 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/53df647d2c624fdc9b487f44b43d4d02 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/53df647d2c624fdc9b487f44b43d4d02 2023-09-29T06:14:52,908 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 18784ff25b183a4d3733801ace5c9d09/C of 18784ff25b183a4d3733801ace5c9d09 into 53df647d2c624fdc9b487f44b43d4d02(size=12.2 K), total size for store is 24.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:14:52,908 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 18784ff25b183a4d3733801ace5c9d09: 2023-09-29T06:14:52,908 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09., storeName=18784ff25b183a4d3733801ace5c9d09/C, priority=13, startTime=1695968092046; duration=0sec 2023-09-29T06:14:52,908 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:52,908 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18784ff25b183a4d3733801ace5c9d09:C 2023-09-29T06:14:52,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:52,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52174 deadline: 1695968152961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:52,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:52,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52146 deadline: 1695968152961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:52,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:52,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968152961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:52,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:52,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52114 deadline: 1695968152963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:52,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:52,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968152966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:53,054 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929b33547c3dac34ae8851cf1220ada2d4f_18784ff25b183a4d3733801ace5c9d09 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929b33547c3dac34ae8851cf1220ada2d4f_18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:14:53,055 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/2481f9a016f14680bec6df08dd11ec30, store: [table=TestAcidGuarantees family=A region=18784ff25b183a4d3733801ace5c9d09] 2023-09-29T06:14:53,056 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/2481f9a016f14680bec6df08dd11ec30 is 175, key is test_row_0/A:col10/1695968092332/Put/seqid=0 2023-09-29T06:14:53,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=113 2023-09-29T06:14:53,070 INFO [Thread-2144 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 113 completed 2023-09-29T06:14:53,071 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:14:53,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2023-09-29T06:14:53,073 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:14:53,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=115 2023-09-29T06:14:53,074 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:14:53,074 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:14:53,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=115 2023-09-29T06:14:53,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2023-09-29T06:14:53,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:53,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. as already flushing 2023-09-29T06:14:53,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:53,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(53): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:53,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:53,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:53,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:53,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968153265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:53,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:53,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52174 deadline: 1695968153265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:53,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:53,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52146 deadline: 1695968153266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:53,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:53,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52114 deadline: 1695968153267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:53,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:53,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968153270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:53,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=115 2023-09-29T06:14:53,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2023-09-29T06:14:53,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:53,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. as already flushing 2023-09-29T06:14:53,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:53,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(53): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:53,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:53,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:53,461 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=196, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/2481f9a016f14680bec6df08dd11ec30 2023-09-29T06:14:53,466 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/37cb5bf2eff24a6ea73c040e115bafe1 is 50, key is test_row_0/B:col10/1695968092332/Put/seqid=0 2023-09-29T06:14:53,470 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/37cb5bf2eff24a6ea73c040e115bafe1 2023-09-29T06:14:53,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/308b460ef3d84139b3edbd06ffe4bf18 is 50, key is test_row_0/C:col10/1695968092332/Put/seqid=0 2023-09-29T06:14:53,479 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/308b460ef3d84139b3edbd06ffe4bf18 2023-09-29T06:14:53,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/2481f9a016f14680bec6df08dd11ec30 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/2481f9a016f14680bec6df08dd11ec30 2023-09-29T06:14:53,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/2481f9a016f14680bec6df08dd11ec30, entries=200, sequenceid=196, filesize=38.8 K 2023-09-29T06:14:53,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/37cb5bf2eff24a6ea73c040e115bafe1 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/37cb5bf2eff24a6ea73c040e115bafe1 2023-09-29T06:14:53,489 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/37cb5bf2eff24a6ea73c040e115bafe1, entries=150, sequenceid=196, filesize=11.8 K 2023-09-29T06:14:53,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/308b460ef3d84139b3edbd06ffe4bf18 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/308b460ef3d84139b3edbd06ffe4bf18 2023-09-29T06:14:53,493 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/308b460ef3d84139b3edbd06ffe4bf18, entries=150, sequenceid=196, filesize=11.8 K 2023-09-29T06:14:53,494 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 18784ff25b183a4d3733801ace5c9d09 in 853ms, sequenceid=196, compaction requested=true 2023-09-29T06:14:53,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 18784ff25b183a4d3733801ace5c9d09: 2023-09-29T06:14:53,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18784ff25b183a4d3733801ace5c9d09:A, priority=-2147483648, current under compaction store size is 1 2023-09-29T06:14:53,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:53,494 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:14:53,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18784ff25b183a4d3733801ace5c9d09:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:14:53,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:53,494 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:14:53,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18784ff25b183a4d3733801ace5c9d09:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:14:53,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:14:53,495 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102250 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:14:53,495 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 18784ff25b183a4d3733801ace5c9d09/A is initiating minor compaction (all files) 2023-09-29T06:14:53,495 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36744 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:14:53,496 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 18784ff25b183a4d3733801ace5c9d09/B is initiating minor compaction (all files) 2023-09-29T06:14:53,496 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 18784ff25b183a4d3733801ace5c9d09/A in TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:53,496 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 18784ff25b183a4d3733801ace5c9d09/B in TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:53,496 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/29bc70f1d05e4dc39efc57e1f9afd723, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/8ff24e9e4b294023b0bf89da2db935a8, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/37cb5bf2eff24a6ea73c040e115bafe1] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp, totalSize=35.9 K 2023-09-29T06:14:53,496 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/92d35f6e6a37480d812984fd295b38c7, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/82b85c2897da4d5e814473b2ae66707f, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/2481f9a016f14680bec6df08dd11ec30] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp, totalSize=99.9 K 2023-09-29T06:14:53,496 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:53,496 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. files: [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/92d35f6e6a37480d812984fd295b38c7, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/82b85c2897da4d5e814473b2ae66707f, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/2481f9a016f14680bec6df08dd11ec30] 2023-09-29T06:14:53,496 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 29bc70f1d05e4dc39efc57e1f9afd723, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1695968090060 2023-09-29T06:14:53,496 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 92d35f6e6a37480d812984fd295b38c7, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1695968090060 2023-09-29T06:14:53,496 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 8ff24e9e4b294023b0bf89da2db935a8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1695968091185 2023-09-29T06:14:53,497 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 82b85c2897da4d5e814473b2ae66707f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1695968091185 2023-09-29T06:14:53,497 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 37cb5bf2eff24a6ea73c040e115bafe1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1695968092329 2023-09-29T06:14:53,497 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 2481f9a016f14680bec6df08dd11ec30, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1695968092329 2023-09-29T06:14:53,502 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=18784ff25b183a4d3733801ace5c9d09] 2023-09-29T06:14:53,503 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20230929ae60b444cca44554aa2889ba42a35154_18784ff25b183a4d3733801ace5c9d09 store=[table=TestAcidGuarantees family=A region=18784ff25b183a4d3733801ace5c9d09] 2023-09-29T06:14:53,503 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18784ff25b183a4d3733801ace5c9d09#B#compaction#388 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:14:53,504 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/7dd2ac36cfb5405a835e9a1f179e6d11 is 50, key is test_row_0/B:col10/1695968092332/Put/seqid=0 2023-09-29T06:14:53,505 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20230929ae60b444cca44554aa2889ba42a35154_18784ff25b183a4d3733801ace5c9d09, store=[table=TestAcidGuarantees family=A region=18784ff25b183a4d3733801ace5c9d09] 2023-09-29T06:14:53,505 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929ae60b444cca44554aa2889ba42a35154_18784ff25b183a4d3733801ace5c9d09 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=18784ff25b183a4d3733801ace5c9d09] 2023-09-29T06:14:53,512 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18784ff25b183a4d3733801ace5c9d09#A#compaction#387 average throughput is 2.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:14:53,512 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/737a22f28f244d1995a4b767e446ba73 is 175, key is test_row_0/A:col10/1695968092332/Put/seqid=0 2023-09-29T06:14:53,515 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/7dd2ac36cfb5405a835e9a1f179e6d11 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/7dd2ac36cfb5405a835e9a1f179e6d11 2023-09-29T06:14:53,519 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 18784ff25b183a4d3733801ace5c9d09/B of 18784ff25b183a4d3733801ace5c9d09 into 7dd2ac36cfb5405a835e9a1f179e6d11(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:14:53,519 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 18784ff25b183a4d3733801ace5c9d09: 2023-09-29T06:14:53,519 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09., storeName=18784ff25b183a4d3733801ace5c9d09/B, priority=13, startTime=1695968093494; duration=0sec 2023-09-29T06:14:53,519 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:14:53,519 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18784ff25b183a4d3733801ace5c9d09:B 2023-09-29T06:14:53,519 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:14:53,520 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36744 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:14:53,520 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 18784ff25b183a4d3733801ace5c9d09/C is initiating minor compaction (all files) 2023-09-29T06:14:53,520 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 18784ff25b183a4d3733801ace5c9d09/C in TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:53,520 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/53df647d2c624fdc9b487f44b43d4d02, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/18554b3c86bf428b9dc43f5b170d1fb3, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/308b460ef3d84139b3edbd06ffe4bf18] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp, totalSize=35.9 K 2023-09-29T06:14:53,520 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 53df647d2c624fdc9b487f44b43d4d02, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1695968090060 2023-09-29T06:14:53,521 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 18554b3c86bf428b9dc43f5b170d1fb3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1695968091185 2023-09-29T06:14:53,521 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 308b460ef3d84139b3edbd06ffe4bf18, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1695968092329 2023-09-29T06:14:53,523 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/737a22f28f244d1995a4b767e446ba73 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/737a22f28f244d1995a4b767e446ba73 2023-09-29T06:14:53,527 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 18784ff25b183a4d3733801ace5c9d09/A of 18784ff25b183a4d3733801ace5c9d09 into 737a22f28f244d1995a4b767e446ba73(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:14:53,527 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 18784ff25b183a4d3733801ace5c9d09: 2023-09-29T06:14:53,527 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09., storeName=18784ff25b183a4d3733801ace5c9d09/A, priority=13, startTime=1695968093494; duration=0sec 2023-09-29T06:14:53,527 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:53,527 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18784ff25b183a4d3733801ace5c9d09:A 2023-09-29T06:14:53,527 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18784ff25b183a4d3733801ace5c9d09#C#compaction#389 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:14:53,528 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/33fcd74ed59047f9a803a530a4893f74 is 50, key is test_row_0/C:col10/1695968092332/Put/seqid=0 2023-09-29T06:14:53,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2023-09-29T06:14:53,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:53,532 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2819): Flushing 18784ff25b183a4d3733801ace5c9d09 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2023-09-29T06:14:53,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 18784ff25b183a4d3733801ace5c9d09, store=A 2023-09-29T06:14:53,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:53,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 18784ff25b183a4d3733801ace5c9d09, store=B 2023-09-29T06:14:53,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:53,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 18784ff25b183a4d3733801ace5c9d09, store=C 2023-09-29T06:14:53,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:53,536 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/33fcd74ed59047f9a803a530a4893f74 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/33fcd74ed59047f9a803a530a4893f74 2023-09-29T06:14:53,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309299d049e1370bc461b82b5e9fc71ff7f97_18784ff25b183a4d3733801ace5c9d09 is 50, key is test_row_0/A:col10/1695968092654/Put/seqid=0 2023-09-29T06:14:53,542 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 18784ff25b183a4d3733801ace5c9d09/C of 18784ff25b183a4d3733801ace5c9d09 into 33fcd74ed59047f9a803a530a4893f74(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:14:53,542 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 18784ff25b183a4d3733801ace5c9d09: 2023-09-29T06:14:53,542 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09., storeName=18784ff25b183a4d3733801ace5c9d09/C, priority=13, startTime=1695968093494; duration=0sec 2023-09-29T06:14:53,542 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:53,542 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18784ff25b183a4d3733801ace5c9d09:C 2023-09-29T06:14:53,547 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309299d049e1370bc461b82b5e9fc71ff7f97_18784ff25b183a4d3733801ace5c9d09 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309299d049e1370bc461b82b5e9fc71ff7f97_18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:14:53,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/3cf15b82c0be486d9b19af4be4db3ccb, store: [table=TestAcidGuarantees family=A region=18784ff25b183a4d3733801ace5c9d09] 2023-09-29T06:14:53,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/3cf15b82c0be486d9b19af4be4db3ccb is 175, key is test_row_0/A:col10/1695968092654/Put/seqid=0 2023-09-29T06:14:53,554 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=209, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/3cf15b82c0be486d9b19af4be4db3ccb 2023-09-29T06:14:53,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/748fcdd70cd2481d92a381b18f81e7e9 is 50, key is test_row_0/B:col10/1695968092654/Put/seqid=0 2023-09-29T06:14:53,569 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/748fcdd70cd2481d92a381b18f81e7e9 2023-09-29T06:14:53,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/b9a8a7b328dd4434a09c4ce18d34faae is 50, key is test_row_0/C:col10/1695968092654/Put/seqid=0 2023-09-29T06:14:53,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=115 2023-09-29T06:14:53,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:14:53,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. as already flushing 2023-09-29T06:14:53,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:53,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968153793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:53,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:53,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52146 deadline: 1695968153794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:53,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:53,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52114 deadline: 1695968153795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:53,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:53,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968153796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:53,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:53,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52174 deadline: 1695968153796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:53,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:53,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968153897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:53,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:53,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52146 deadline: 1695968153900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:53,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:53,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52114 deadline: 1695968153900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:53,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:53,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968153900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:53,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:53,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52174 deadline: 1695968153901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:53,979 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/b9a8a7b328dd4434a09c4ce18d34faae 2023-09-29T06:14:53,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/3cf15b82c0be486d9b19af4be4db3ccb as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/3cf15b82c0be486d9b19af4be4db3ccb 2023-09-29T06:14:53,986 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/3cf15b82c0be486d9b19af4be4db3ccb, entries=150, sequenceid=209, filesize=30.4 K 2023-09-29T06:14:53,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/748fcdd70cd2481d92a381b18f81e7e9 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/748fcdd70cd2481d92a381b18f81e7e9 2023-09-29T06:14:53,990 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/748fcdd70cd2481d92a381b18f81e7e9, entries=150, sequenceid=209, filesize=11.8 K 2023-09-29T06:14:53,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/b9a8a7b328dd4434a09c4ce18d34faae as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/b9a8a7b328dd4434a09c4ce18d34faae 2023-09-29T06:14:53,994 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/b9a8a7b328dd4434a09c4ce18d34faae, entries=150, sequenceid=209, filesize=11.8 K 2023-09-29T06:14:53,995 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3022): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for 18784ff25b183a4d3733801ace5c9d09 in 463ms, sequenceid=209, compaction requested=false 2023-09-29T06:14:53,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2520): Flush status journal for 18784ff25b183a4d3733801ace5c9d09: 2023-09-29T06:14:53,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:53,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=116 2023-09-29T06:14:53,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=116 2023-09-29T06:14:53,998 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=116, resume processing ppid=115 2023-09-29T06:14:53,998 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=116, ppid=115, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 922 msec 2023-09-29T06:14:53,999 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=115, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 927 msec 2023-09-29T06:14:54,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:14:54,104 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 18784ff25b183a4d3733801ace5c9d09 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2023-09-29T06:14:54,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 18784ff25b183a4d3733801ace5c9d09, store=A 2023-09-29T06:14:54,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:54,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 18784ff25b183a4d3733801ace5c9d09, store=B 2023-09-29T06:14:54,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:54,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 18784ff25b183a4d3733801ace5c9d09, store=C 2023-09-29T06:14:54,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:54,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309295d130c19df8140b6a3e6a475cd724411_18784ff25b183a4d3733801ace5c9d09 is 50, key is test_row_0/A:col10/1695968093795/Put/seqid=0 2023-09-29T06:14:54,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:54,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968154108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:54,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:54,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52146 deadline: 1695968154109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:54,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:54,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52174 deadline: 1695968154110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:54,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:54,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52114 deadline: 1695968154110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:54,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:54,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968154111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:54,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=115 2023-09-29T06:14:54,177 INFO [Thread-2144 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 115 completed 2023-09-29T06:14:54,178 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:14:54,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2023-09-29T06:14:54,181 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:14:54,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=117 2023-09-29T06:14:54,181 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:14:54,181 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:14:54,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:54,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968154211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:54,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:54,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:54,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52146 deadline: 1695968154214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:54,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52174 deadline: 1695968154214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:54,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:54,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:54,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52114 deadline: 1695968154216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:54,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968154216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:54,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=117 2023-09-29T06:14:54,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2023-09-29T06:14:54,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:54,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. as already flushing 2023-09-29T06:14:54,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:54,334 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(53): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:54,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:54,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:54,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:54,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968154415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:54,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:54,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52174 deadline: 1695968154420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:54,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:54,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52146 deadline: 1695968154421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:54,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:54,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52114 deadline: 1695968154421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:54,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:54,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968154421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:54,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=117 2023-09-29T06:14:54,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2023-09-29T06:14:54,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:54,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. as already flushing 2023-09-29T06:14:54,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:54,487 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(53): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:54,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:54,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:54,517 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309295d130c19df8140b6a3e6a475cd724411_18784ff25b183a4d3733801ace5c9d09 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309295d130c19df8140b6a3e6a475cd724411_18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:14:54,518 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/6f469fa0c8f54a1bb02b9fea054b529e, store: [table=TestAcidGuarantees family=A region=18784ff25b183a4d3733801ace5c9d09] 2023-09-29T06:14:54,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/6f469fa0c8f54a1bb02b9fea054b529e is 175, key is test_row_0/A:col10/1695968093795/Put/seqid=0 2023-09-29T06:14:54,523 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=238, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/6f469fa0c8f54a1bb02b9fea054b529e 2023-09-29T06:14:54,542 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/ffd1798c93d742969dbce76cc498efbf is 50, key is test_row_0/B:col10/1695968093795/Put/seqid=0 2023-09-29T06:14:54,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2023-09-29T06:14:54,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:54,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. as already flushing 2023-09-29T06:14:54,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:54,640 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(53): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:54,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:54,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:54,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:54,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968154721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:54,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:54,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52174 deadline: 1695968154724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:54,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:54,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52114 deadline: 1695968154724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:54,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:54,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:54,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52146 deadline: 1695968154725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:54,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968154725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:54,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=117 2023-09-29T06:14:54,792 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2023-09-29T06:14:54,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:54,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. as already flushing 2023-09-29T06:14:54,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:54,792 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(53): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:54,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:54,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:54,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2023-09-29T06:14:54,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:54,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. as already flushing 2023-09-29T06:14:54,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:54,945 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(53): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:54,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:54,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:54,952 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/ffd1798c93d742969dbce76cc498efbf 2023-09-29T06:14:54,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/2c4316ff16874f8ebf84001486298493 is 50, key is test_row_0/C:col10/1695968093795/Put/seqid=0 2023-09-29T06:14:54,963 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/2c4316ff16874f8ebf84001486298493 2023-09-29T06:14:54,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/6f469fa0c8f54a1bb02b9fea054b529e as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/6f469fa0c8f54a1bb02b9fea054b529e 2023-09-29T06:14:54,970 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/6f469fa0c8f54a1bb02b9fea054b529e, entries=200, sequenceid=238, filesize=38.8 K 2023-09-29T06:14:54,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/ffd1798c93d742969dbce76cc498efbf as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/ffd1798c93d742969dbce76cc498efbf 2023-09-29T06:14:54,973 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/ffd1798c93d742969dbce76cc498efbf, entries=150, sequenceid=238, filesize=11.8 K 2023-09-29T06:14:54,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/2c4316ff16874f8ebf84001486298493 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/2c4316ff16874f8ebf84001486298493 2023-09-29T06:14:54,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/2c4316ff16874f8ebf84001486298493, entries=150, sequenceid=238, filesize=11.8 K 2023-09-29T06:14:54,978 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 18784ff25b183a4d3733801ace5c9d09 in 875ms, sequenceid=238, compaction requested=true 2023-09-29T06:14:54,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 18784ff25b183a4d3733801ace5c9d09: 2023-09-29T06:14:54,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18784ff25b183a4d3733801ace5c9d09:A, priority=-2147483648, current under compaction store size is 1 2023-09-29T06:14:54,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:54,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18784ff25b183a4d3733801ace5c9d09:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:14:54,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:54,978 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:14:54,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18784ff25b183a4d3733801ace5c9d09:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:14:54,978 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:14:54,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:14:54,979 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36846 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:14:54,979 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 18784ff25b183a4d3733801ace5c9d09/B is initiating minor compaction (all files) 2023-09-29T06:14:54,979 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 18784ff25b183a4d3733801ace5c9d09/B in TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:54,979 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102352 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:14:54,979 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 18784ff25b183a4d3733801ace5c9d09/A is initiating minor compaction (all files) 2023-09-29T06:14:54,979 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/7dd2ac36cfb5405a835e9a1f179e6d11, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/748fcdd70cd2481d92a381b18f81e7e9, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/ffd1798c93d742969dbce76cc498efbf] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp, totalSize=36.0 K 2023-09-29T06:14:54,979 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 18784ff25b183a4d3733801ace5c9d09/A in TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:54,979 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/737a22f28f244d1995a4b767e446ba73, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/3cf15b82c0be486d9b19af4be4db3ccb, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/6f469fa0c8f54a1bb02b9fea054b529e] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp, totalSize=100.0 K 2023-09-29T06:14:54,979 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:54,979 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. files: [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/737a22f28f244d1995a4b767e446ba73, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/3cf15b82c0be486d9b19af4be4db3ccb, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/6f469fa0c8f54a1bb02b9fea054b529e] 2023-09-29T06:14:54,979 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 7dd2ac36cfb5405a835e9a1f179e6d11, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1695968092329 2023-09-29T06:14:54,980 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 748fcdd70cd2481d92a381b18f81e7e9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1695968092647 2023-09-29T06:14:54,980 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 737a22f28f244d1995a4b767e446ba73, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1695968092329 2023-09-29T06:14:54,980 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting ffd1798c93d742969dbce76cc498efbf, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1695968093795 2023-09-29T06:14:54,980 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 3cf15b82c0be486d9b19af4be4db3ccb, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1695968092647 2023-09-29T06:14:54,981 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 6f469fa0c8f54a1bb02b9fea054b529e, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1695968093794 2023-09-29T06:14:54,985 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=18784ff25b183a4d3733801ace5c9d09] 2023-09-29T06:14:54,986 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18784ff25b183a4d3733801ace5c9d09#B#compaction#396 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:14:54,987 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2023092964090ff861fb4267a1266159dc3fed9f_18784ff25b183a4d3733801ace5c9d09 store=[table=TestAcidGuarantees family=A region=18784ff25b183a4d3733801ace5c9d09] 2023-09-29T06:14:54,987 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/2e23c0d78c914681ab62094626265e44 is 50, key is test_row_0/B:col10/1695968093795/Put/seqid=0 2023-09-29T06:14:54,988 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2023092964090ff861fb4267a1266159dc3fed9f_18784ff25b183a4d3733801ace5c9d09, store=[table=TestAcidGuarantees family=A region=18784ff25b183a4d3733801ace5c9d09] 2023-09-29T06:14:54,988 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023092964090ff861fb4267a1266159dc3fed9f_18784ff25b183a4d3733801ace5c9d09 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=18784ff25b183a4d3733801ace5c9d09] 2023-09-29T06:14:55,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2023-09-29T06:14:55,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:55,098 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2819): Flushing 18784ff25b183a4d3733801ace5c9d09 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2023-09-29T06:14:55,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 18784ff25b183a4d3733801ace5c9d09, store=A 2023-09-29T06:14:55,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:55,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 18784ff25b183a4d3733801ace5c9d09, store=B 2023-09-29T06:14:55,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:55,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 18784ff25b183a4d3733801ace5c9d09, store=C 2023-09-29T06:14:55,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:55,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929ef0a407fb5b54c35b2664ef33a158dc7_18784ff25b183a4d3733801ace5c9d09 is 50, key is test_row_0/A:col10/1695968094108/Put/seqid=0 2023-09-29T06:14:55,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:14:55,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. as already flushing 2023-09-29T06:14:55,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=117 2023-09-29T06:14:55,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:55,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968155284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:55,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:55,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968155289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:55,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:55,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52114 deadline: 1695968155290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:55,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:55,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52174 deadline: 1695968155291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:55,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:55,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52146 deadline: 1695968155292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:55,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:55,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968155393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:55,396 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/2e23c0d78c914681ab62094626265e44 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/2e23c0d78c914681ab62094626265e44 2023-09-29T06:14:55,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:55,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968155395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:55,401 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18784ff25b183a4d3733801ace5c9d09#A#compaction#397 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:14:55,401 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 18784ff25b183a4d3733801ace5c9d09/B of 18784ff25b183a4d3733801ace5c9d09 into 2e23c0d78c914681ab62094626265e44(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:14:55,401 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 18784ff25b183a4d3733801ace5c9d09: 2023-09-29T06:14:55,401 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09., storeName=18784ff25b183a4d3733801ace5c9d09/B, priority=13, startTime=1695968094978; duration=0sec 2023-09-29T06:14:55,401 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:14:55,401 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18784ff25b183a4d3733801ace5c9d09:B 2023-09-29T06:14:55,401 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:14:55,402 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/51548e1b977a48a7a2c27b81d8fafaf9 is 175, key is test_row_0/A:col10/1695968093795/Put/seqid=0 2023-09-29T06:14:55,402 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36846 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:14:55,402 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 18784ff25b183a4d3733801ace5c9d09/C is initiating minor compaction (all files) 2023-09-29T06:14:55,402 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 18784ff25b183a4d3733801ace5c9d09/C in TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:55,402 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/33fcd74ed59047f9a803a530a4893f74, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/b9a8a7b328dd4434a09c4ce18d34faae, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/2c4316ff16874f8ebf84001486298493] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp, totalSize=36.0 K 2023-09-29T06:14:55,403 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 33fcd74ed59047f9a803a530a4893f74, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1695968092329 2023-09-29T06:14:55,403 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting b9a8a7b328dd4434a09c4ce18d34faae, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1695968092647 2023-09-29T06:14:55,403 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 2c4316ff16874f8ebf84001486298493, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1695968093795 2023-09-29T06:14:55,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:55,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52114 deadline: 1695968155399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:55,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:55,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52174 deadline: 1695968155401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:55,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:55,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52146 deadline: 1695968155402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:55,409 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/51548e1b977a48a7a2c27b81d8fafaf9 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/51548e1b977a48a7a2c27b81d8fafaf9 2023-09-29T06:14:55,410 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18784ff25b183a4d3733801ace5c9d09#C#compaction#399 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:14:55,410 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/7e79cbd361a54916b8d7908d537a1d58 is 50, key is test_row_0/C:col10/1695968093795/Put/seqid=0 2023-09-29T06:14:55,413 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 18784ff25b183a4d3733801ace5c9d09/A of 18784ff25b183a4d3733801ace5c9d09 into 51548e1b977a48a7a2c27b81d8fafaf9(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:14:55,414 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 18784ff25b183a4d3733801ace5c9d09: 2023-09-29T06:14:55,414 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09., storeName=18784ff25b183a4d3733801ace5c9d09/A, priority=13, startTime=1695968094978; duration=0sec 2023-09-29T06:14:55,414 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:55,414 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18784ff25b183a4d3733801ace5c9d09:A 2023-09-29T06:14:55,417 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/7e79cbd361a54916b8d7908d537a1d58 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/7e79cbd361a54916b8d7908d537a1d58 2023-09-29T06:14:55,421 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 18784ff25b183a4d3733801ace5c9d09/C of 18784ff25b183a4d3733801ace5c9d09 into 7e79cbd361a54916b8d7908d537a1d58(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:14:55,421 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 18784ff25b183a4d3733801ace5c9d09: 2023-09-29T06:14:55,421 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09., storeName=18784ff25b183a4d3733801ace5c9d09/C, priority=13, startTime=1695968094978; duration=0sec 2023-09-29T06:14:55,421 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:55,421 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18784ff25b183a4d3733801ace5c9d09:C 2023-09-29T06:14:55,512 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929ef0a407fb5b54c35b2664ef33a158dc7_18784ff25b183a4d3733801ace5c9d09 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929ef0a407fb5b54c35b2664ef33a158dc7_18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:14:55,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/b0ea1640bb584e5a99123c59b3cd7cb4, store: [table=TestAcidGuarantees family=A region=18784ff25b183a4d3733801ace5c9d09] 2023-09-29T06:14:55,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/b0ea1640bb584e5a99123c59b3cd7cb4 is 175, key is test_row_0/A:col10/1695968094108/Put/seqid=0 2023-09-29T06:14:55,530 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=246, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/b0ea1640bb584e5a99123c59b3cd7cb4 2023-09-29T06:14:55,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/166878c24b93473d9a565df336c4a7e8 is 50, key is test_row_0/B:col10/1695968094108/Put/seqid=0 2023-09-29T06:14:55,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:55,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968155594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:55,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:55,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968155600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:55,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:55,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52114 deadline: 1695968155605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:55,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:55,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52174 deadline: 1695968155608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:55,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:55,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52146 deadline: 1695968155609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:55,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:55,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968155897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:55,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:55,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968155903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:55,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:55,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52114 deadline: 1695968155910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:55,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:55,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52174 deadline: 1695968155914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:55,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:55,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52146 deadline: 1695968155916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:55,944 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/166878c24b93473d9a565df336c4a7e8 2023-09-29T06:14:55,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/6f689318790145a49b49c40761ee9175 is 50, key is test_row_0/C:col10/1695968094108/Put/seqid=0 2023-09-29T06:14:56,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=117 2023-09-29T06:14:56,354 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/6f689318790145a49b49c40761ee9175 2023-09-29T06:14:56,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/b0ea1640bb584e5a99123c59b3cd7cb4 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/b0ea1640bb584e5a99123c59b3cd7cb4 2023-09-29T06:14:56,361 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/b0ea1640bb584e5a99123c59b3cd7cb4, entries=150, sequenceid=246, filesize=30.4 K 2023-09-29T06:14:56,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/166878c24b93473d9a565df336c4a7e8 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/166878c24b93473d9a565df336c4a7e8 2023-09-29T06:14:56,365 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/166878c24b93473d9a565df336c4a7e8, entries=150, sequenceid=246, filesize=11.8 K 2023-09-29T06:14:56,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/6f689318790145a49b49c40761ee9175 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/6f689318790145a49b49c40761ee9175 2023-09-29T06:14:56,369 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/6f689318790145a49b49c40761ee9175, entries=150, sequenceid=246, filesize=11.8 K 2023-09-29T06:14:56,369 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3022): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for 18784ff25b183a4d3733801ace5c9d09 in 1272ms, sequenceid=246, compaction requested=false 2023-09-29T06:14:56,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2520): Flush status journal for 18784ff25b183a4d3733801ace5c9d09: 2023-09-29T06:14:56,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:56,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=118 2023-09-29T06:14:56,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=118 2023-09-29T06:14:56,371 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=118, resume processing ppid=117 2023-09-29T06:14:56,371 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=118, ppid=117, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1890 sec 2023-09-29T06:14:56,373 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=117, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 2.1940 sec 2023-09-29T06:14:56,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:14:56,404 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 18784ff25b183a4d3733801ace5c9d09 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2023-09-29T06:14:56,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 18784ff25b183a4d3733801ace5c9d09, store=A 2023-09-29T06:14:56,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:56,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 18784ff25b183a4d3733801ace5c9d09, store=B 2023-09-29T06:14:56,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:56,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 18784ff25b183a4d3733801ace5c9d09, store=C 2023-09-29T06:14:56,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:56,409 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929522f2882d0c446368e5d6d597f24a663_18784ff25b183a4d3733801ace5c9d09 is 50, key is test_row_0/A:col10/1695968095282/Put/seqid=0 2023-09-29T06:14:56,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:56,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52114 deadline: 1695968156414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:56,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:56,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968156415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:56,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:56,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968156416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:56,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:56,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52146 deadline: 1695968156419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:56,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:56,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52174 deadline: 1695968156421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:56,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:56,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968156518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:56,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:56,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968156521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:56,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:56,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968156722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:56,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:56,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968156725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:56,816 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929522f2882d0c446368e5d6d597f24a663_18784ff25b183a4d3733801ace5c9d09 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929522f2882d0c446368e5d6d597f24a663_18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:14:56,817 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/46ea766e2a214216bde59ca60c651b77, store: [table=TestAcidGuarantees family=A region=18784ff25b183a4d3733801ace5c9d09] 2023-09-29T06:14:56,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/46ea766e2a214216bde59ca60c651b77 is 175, key is test_row_0/A:col10/1695968095282/Put/seqid=0 2023-09-29T06:14:57,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:57,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968157027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:57,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:57,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968157033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:57,221 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=278, memsize=58.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/46ea766e2a214216bde59ca60c651b77 2023-09-29T06:14:57,228 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/39fc0dd678c64789bfade735e6f6818f is 50, key is test_row_0/B:col10/1695968095282/Put/seqid=0 2023-09-29T06:14:57,231 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/39fc0dd678c64789bfade735e6f6818f 2023-09-29T06:14:57,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/a2a2c21eb9da475a8adc3f1306422527 is 50, key is test_row_0/C:col10/1695968095282/Put/seqid=0 2023-09-29T06:14:57,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:57,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52114 deadline: 1695968157423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:57,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:57,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52146 deadline: 1695968157424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:57,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:57,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52174 deadline: 1695968157432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:57,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:57,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968157533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:57,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:57,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968157536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:57,640 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/a2a2c21eb9da475a8adc3f1306422527 2023-09-29T06:14:57,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/46ea766e2a214216bde59ca60c651b77 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/46ea766e2a214216bde59ca60c651b77 2023-09-29T06:14:57,648 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/46ea766e2a214216bde59ca60c651b77, entries=200, sequenceid=278, filesize=39.0 K 2023-09-29T06:14:57,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/39fc0dd678c64789bfade735e6f6818f as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/39fc0dd678c64789bfade735e6f6818f 2023-09-29T06:14:57,651 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/39fc0dd678c64789bfade735e6f6818f, entries=150, sequenceid=278, filesize=12.0 K 2023-09-29T06:14:57,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/a2a2c21eb9da475a8adc3f1306422527 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/a2a2c21eb9da475a8adc3f1306422527 2023-09-29T06:14:57,656 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/a2a2c21eb9da475a8adc3f1306422527, entries=150, sequenceid=278, filesize=12.0 K 2023-09-29T06:14:57,656 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for 18784ff25b183a4d3733801ace5c9d09 in 1252ms, sequenceid=278, compaction requested=true 2023-09-29T06:14:57,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 18784ff25b183a4d3733801ace5c9d09: 2023-09-29T06:14:57,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18784ff25b183a4d3733801ace5c9d09:A, priority=-2147483648, current under compaction store size is 1 2023-09-29T06:14:57,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:57,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18784ff25b183a4d3733801ace5c9d09:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:14:57,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:14:57,656 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:14:57,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18784ff25b183a4d3733801ace5c9d09:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:14:57,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2023-09-29T06:14:57,657 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:14:57,658 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102654 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:14:57,658 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37098 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:14:57,658 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 18784ff25b183a4d3733801ace5c9d09/A is initiating minor compaction (all files) 2023-09-29T06:14:57,658 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 18784ff25b183a4d3733801ace5c9d09/B is initiating minor compaction (all files) 2023-09-29T06:14:57,658 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 18784ff25b183a4d3733801ace5c9d09/A in TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:57,658 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 18784ff25b183a4d3733801ace5c9d09/B in TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:57,658 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/51548e1b977a48a7a2c27b81d8fafaf9, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/b0ea1640bb584e5a99123c59b3cd7cb4, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/46ea766e2a214216bde59ca60c651b77] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp, totalSize=100.2 K 2023-09-29T06:14:57,658 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/2e23c0d78c914681ab62094626265e44, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/166878c24b93473d9a565df336c4a7e8, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/39fc0dd678c64789bfade735e6f6818f] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp, totalSize=36.2 K 2023-09-29T06:14:57,658 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:57,658 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. files: [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/51548e1b977a48a7a2c27b81d8fafaf9, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/b0ea1640bb584e5a99123c59b3cd7cb4, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/46ea766e2a214216bde59ca60c651b77] 2023-09-29T06:14:57,658 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 2e23c0d78c914681ab62094626265e44, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1695968093795 2023-09-29T06:14:57,662 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 166878c24b93473d9a565df336c4a7e8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1695968094108 2023-09-29T06:14:57,662 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 51548e1b977a48a7a2c27b81d8fafaf9, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1695968093795 2023-09-29T06:14:57,662 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting b0ea1640bb584e5a99123c59b3cd7cb4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1695968094108 2023-09-29T06:14:57,662 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 39fc0dd678c64789bfade735e6f6818f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1695968095282 2023-09-29T06:14:57,663 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 46ea766e2a214216bde59ca60c651b77, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1695968095282 2023-09-29T06:14:57,668 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=18784ff25b183a4d3733801ace5c9d09] 2023-09-29T06:14:57,669 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18784ff25b183a4d3733801ace5c9d09#B#compaction#405 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:14:57,670 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/507fb535c9c343a7bc119ca6ee986318 is 50, key is test_row_0/B:col10/1695968095282/Put/seqid=0 2023-09-29T06:14:57,670 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202309293d2157bee2094a06a4a67a207e8a3ba6_18784ff25b183a4d3733801ace5c9d09 store=[table=TestAcidGuarantees family=A region=18784ff25b183a4d3733801ace5c9d09] 2023-09-29T06:14:57,671 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202309293d2157bee2094a06a4a67a207e8a3ba6_18784ff25b183a4d3733801ace5c9d09, store=[table=TestAcidGuarantees family=A region=18784ff25b183a4d3733801ace5c9d09] 2023-09-29T06:14:57,671 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309293d2157bee2094a06a4a67a207e8a3ba6_18784ff25b183a4d3733801ace5c9d09 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=18784ff25b183a4d3733801ace5c9d09] 2023-09-29T06:14:57,676 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18784ff25b183a4d3733801ace5c9d09#A#compaction#406 average throughput is 3.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:14:57,677 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/69a36a3b3c36467caa6890191231b396 is 175, key is test_row_0/A:col10/1695968095282/Put/seqid=0 2023-09-29T06:14:57,686 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/69a36a3b3c36467caa6890191231b396 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/69a36a3b3c36467caa6890191231b396 2023-09-29T06:14:57,690 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 18784ff25b183a4d3733801ace5c9d09/A of 18784ff25b183a4d3733801ace5c9d09 into 69a36a3b3c36467caa6890191231b396(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:14:57,690 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 18784ff25b183a4d3733801ace5c9d09: 2023-09-29T06:14:57,690 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09., storeName=18784ff25b183a4d3733801ace5c9d09/A, priority=13, startTime=1695968097656; duration=0sec 2023-09-29T06:14:57,690 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:14:57,690 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18784ff25b183a4d3733801ace5c9d09:A 2023-09-29T06:14:57,691 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:14:57,691 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37098 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:14:57,692 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 18784ff25b183a4d3733801ace5c9d09/C is initiating minor compaction (all files) 2023-09-29T06:14:57,692 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 18784ff25b183a4d3733801ace5c9d09/C in TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:57,692 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/7e79cbd361a54916b8d7908d537a1d58, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/6f689318790145a49b49c40761ee9175, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/a2a2c21eb9da475a8adc3f1306422527] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp, totalSize=36.2 K 2023-09-29T06:14:57,692 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 7e79cbd361a54916b8d7908d537a1d58, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1695968093795 2023-09-29T06:14:57,692 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 6f689318790145a49b49c40761ee9175, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1695968094108 2023-09-29T06:14:57,693 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting a2a2c21eb9da475a8adc3f1306422527, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1695968095282 2023-09-29T06:14:57,698 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18784ff25b183a4d3733801ace5c9d09#C#compaction#407 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:14:57,699 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/f9bb3f8f09624606bd341f1cd39d918f is 50, key is test_row_0/C:col10/1695968095282/Put/seqid=0 2023-09-29T06:14:57,718 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/f9bb3f8f09624606bd341f1cd39d918f as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/f9bb3f8f09624606bd341f1cd39d918f 2023-09-29T06:14:57,722 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 18784ff25b183a4d3733801ace5c9d09/C of 18784ff25b183a4d3733801ace5c9d09 into f9bb3f8f09624606bd341f1cd39d918f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:14:57,722 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 18784ff25b183a4d3733801ace5c9d09: 2023-09-29T06:14:57,722 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09., storeName=18784ff25b183a4d3733801ace5c9d09/C, priority=13, startTime=1695968097656; duration=0sec 2023-09-29T06:14:57,722 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:57,722 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18784ff25b183a4d3733801ace5c9d09:C 2023-09-29T06:14:58,078 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/507fb535c9c343a7bc119ca6ee986318 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/507fb535c9c343a7bc119ca6ee986318 2023-09-29T06:14:58,082 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 18784ff25b183a4d3733801ace5c9d09/B of 18784ff25b183a4d3733801ace5c9d09 into 507fb535c9c343a7bc119ca6ee986318(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:14:58,082 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 18784ff25b183a4d3733801ace5c9d09: 2023-09-29T06:14:58,082 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09., storeName=18784ff25b183a4d3733801ace5c9d09/B, priority=13, startTime=1695968097656; duration=0sec 2023-09-29T06:14:58,082 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:58,082 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18784ff25b183a4d3733801ace5c9d09:B 2023-09-29T06:14:58,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=117 2023-09-29T06:14:58,286 INFO [Thread-2144 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 117 completed 2023-09-29T06:14:58,287 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:14:58,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2023-09-29T06:14:58,289 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:14:58,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=119 2023-09-29T06:14:58,290 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:14:58,290 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:14:58,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=119 2023-09-29T06:14:58,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2023-09-29T06:14:58,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:58,443 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2819): Flushing 18784ff25b183a4d3733801ace5c9d09 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2023-09-29T06:14:58,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 18784ff25b183a4d3733801ace5c9d09, store=A 2023-09-29T06:14:58,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:58,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 18784ff25b183a4d3733801ace5c9d09, store=B 2023-09-29T06:14:58,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:58,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 18784ff25b183a4d3733801ace5c9d09, store=C 2023-09-29T06:14:58,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:58,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023092916904332e2e64d299cb1437967f6664f_18784ff25b183a4d3733801ace5c9d09 is 50, key is test_row_0/A:col10/1695968096414/Put/seqid=0 2023-09-29T06:14:58,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:14:58,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. as already flushing 2023-09-29T06:14:58,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=119 2023-09-29T06:14:58,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:58,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968158622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:58,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:58,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968158625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:58,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:58,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968158727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:58,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:58,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968158730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:58,868 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023092916904332e2e64d299cb1437967f6664f_18784ff25b183a4d3733801ace5c9d09 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023092916904332e2e64d299cb1437967f6664f_18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:14:58,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/9cfe148a36c24a208a3be510cd09596d, store: [table=TestAcidGuarantees family=A region=18784ff25b183a4d3733801ace5c9d09] 2023-09-29T06:14:58,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/9cfe148a36c24a208a3be510cd09596d is 175, key is test_row_0/A:col10/1695968096414/Put/seqid=0 2023-09-29T06:14:58,874 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=288, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/9cfe148a36c24a208a3be510cd09596d 2023-09-29T06:14:58,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/3db2139efb4e480e94fb3fa12097357e is 50, key is test_row_0/B:col10/1695968096414/Put/seqid=0 2023-09-29T06:14:58,885 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/3db2139efb4e480e94fb3fa12097357e 2023-09-29T06:14:58,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/82a1c1396ef84ec082abffec18a157e6 is 50, key is test_row_0/C:col10/1695968096414/Put/seqid=0 2023-09-29T06:14:58,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=119 2023-09-29T06:14:58,896 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/82a1c1396ef84ec082abffec18a157e6 2023-09-29T06:14:58,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/9cfe148a36c24a208a3be510cd09596d as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/9cfe148a36c24a208a3be510cd09596d 2023-09-29T06:14:58,903 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/9cfe148a36c24a208a3be510cd09596d, entries=100, sequenceid=288, filesize=22.0 K 2023-09-29T06:14:58,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/3db2139efb4e480e94fb3fa12097357e as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/3db2139efb4e480e94fb3fa12097357e 2023-09-29T06:14:58,907 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/3db2139efb4e480e94fb3fa12097357e, entries=100, sequenceid=288, filesize=9.6 K 2023-09-29T06:14:58,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/82a1c1396ef84ec082abffec18a157e6 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/82a1c1396ef84ec082abffec18a157e6 2023-09-29T06:14:58,911 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/82a1c1396ef84ec082abffec18a157e6, entries=100, sequenceid=288, filesize=9.6 K 2023-09-29T06:14:58,911 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3022): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=174.43 KB/178620 for 18784ff25b183a4d3733801ace5c9d09 in 468ms, sequenceid=288, compaction requested=false 2023-09-29T06:14:58,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2520): Flush status journal for 18784ff25b183a4d3733801ace5c9d09: 2023-09-29T06:14:58,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:58,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=120 2023-09-29T06:14:58,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=120 2023-09-29T06:14:58,914 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=120, resume processing ppid=119 2023-09-29T06:14:58,914 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=120, ppid=119, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 623 msec 2023-09-29T06:14:58,915 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=119, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 627 msec 2023-09-29T06:14:58,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:14:58,934 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 18784ff25b183a4d3733801ace5c9d09 3/3 column families, dataSize=181.14 KB heapSize=475.36 KB 2023-09-29T06:14:58,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 18784ff25b183a4d3733801ace5c9d09, store=A 2023-09-29T06:14:58,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:58,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 18784ff25b183a4d3733801ace5c9d09, store=B 2023-09-29T06:14:58,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:58,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 18784ff25b183a4d3733801ace5c9d09, store=C 2023-09-29T06:14:58,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:58,939 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309291d00de74d3c34ca8b0ed35becefcb780_18784ff25b183a4d3733801ace5c9d09 is 50, key is test_row_0/A:col10/1695968098611/Put/seqid=0 2023-09-29T06:14:58,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:58,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968158940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:58,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:58,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968158941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:58,947 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309291d00de74d3c34ca8b0ed35becefcb780_18784ff25b183a4d3733801ace5c9d09 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309291d00de74d3c34ca8b0ed35becefcb780_18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:14:58,947 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/e761274211cc4664a7507c3461a52b73, store: [table=TestAcidGuarantees family=A region=18784ff25b183a4d3733801ace5c9d09] 2023-09-29T06:14:58,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/e761274211cc4664a7507c3461a52b73 is 175, key is test_row_0/A:col10/1695968098611/Put/seqid=0 2023-09-29T06:14:59,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:59,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968159046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:59,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:59,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968159047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:59,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:59,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968159249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:59,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:59,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968159250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:59,352 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=318, memsize=60.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/e761274211cc4664a7507c3461a52b73 2023-09-29T06:14:59,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/3aa93e87805d40dd9fcc11d990583ebe is 50, key is test_row_0/B:col10/1695968098611/Put/seqid=0 2023-09-29T06:14:59,362 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/3aa93e87805d40dd9fcc11d990583ebe 2023-09-29T06:14:59,369 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/8e06210a534441c59a1c5f9f78138fb4 is 50, key is test_row_0/C:col10/1695968098611/Put/seqid=0 2023-09-29T06:14:59,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=119 2023-09-29T06:14:59,393 INFO [Thread-2144 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 119 completed 2023-09-29T06:14:59,394 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:14:59,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2023-09-29T06:14:59,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=121 2023-09-29T06:14:59,396 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:14:59,397 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:14:59,397 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:14:59,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:59,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52146 deadline: 1695968159443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:59,446 DEBUG [Thread-2134 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4153 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_0' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09., hostname=jenkins-hbase3.apache.org,39783,1695967978138, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:572) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:559) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:197) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:218) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:750) 2023-09-29T06:14:59,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:59,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52114 deadline: 1695968159444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:59,449 DEBUG [Thread-2138 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4158 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_0' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09., hostname=jenkins-hbase3.apache.org,39783,1695967978138, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:572) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:559) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:197) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:218) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:750) 2023-09-29T06:14:59,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:59,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52174 deadline: 1695968159455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:59,457 DEBUG [Thread-2136 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4166 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_0' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09., hostname=jenkins-hbase3.apache.org,39783,1695967978138, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:572) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:559) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:197) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:218) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:750) 2023-09-29T06:14:59,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=121 2023-09-29T06:14:59,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2023-09-29T06:14:59,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:59,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. as already flushing 2023-09-29T06:14:59,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:59,550 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(53): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:59,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:59,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:59,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:59,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968159553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:59,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:14:59,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968159554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:14:59,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=121 2023-09-29T06:14:59,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2023-09-29T06:14:59,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:59,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. as already flushing 2023-09-29T06:14:59,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:59,703 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(53): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:59,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:59,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:14:59,773 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/8e06210a534441c59a1c5f9f78138fb4 2023-09-29T06:14:59,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/e761274211cc4664a7507c3461a52b73 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/e761274211cc4664a7507c3461a52b73 2023-09-29T06:14:59,798 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/e761274211cc4664a7507c3461a52b73, entries=200, sequenceid=318, filesize=39.0 K 2023-09-29T06:14:59,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/3aa93e87805d40dd9fcc11d990583ebe as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/3aa93e87805d40dd9fcc11d990583ebe 2023-09-29T06:14:59,803 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/3aa93e87805d40dd9fcc11d990583ebe, entries=150, sequenceid=318, filesize=12.0 K 2023-09-29T06:14:59,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/8e06210a534441c59a1c5f9f78138fb4 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/8e06210a534441c59a1c5f9f78138fb4 2023-09-29T06:14:59,806 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/8e06210a534441c59a1c5f9f78138fb4, entries=150, sequenceid=318, filesize=12.0 K 2023-09-29T06:14:59,807 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=20.13 KB/20610 for 18784ff25b183a4d3733801ace5c9d09 in 873ms, sequenceid=318, compaction requested=true 2023-09-29T06:14:59,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 18784ff25b183a4d3733801ace5c9d09: 2023-09-29T06:14:59,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18784ff25b183a4d3733801ace5c9d09:A, priority=-2147483648, current under compaction store size is 1 2023-09-29T06:14:59,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:59,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18784ff25b183a4d3733801ace5c9d09:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:14:59,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:14:59,807 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:14:59,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18784ff25b183a4d3733801ace5c9d09:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:14:59,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2023-09-29T06:14:59,809 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:14:59,813 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94362 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:14:59,813 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 18784ff25b183a4d3733801ace5c9d09/A is initiating minor compaction (all files) 2023-09-29T06:14:59,813 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 18784ff25b183a4d3733801ace5c9d09/A in TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:59,813 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/69a36a3b3c36467caa6890191231b396, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/9cfe148a36c24a208a3be510cd09596d, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/e761274211cc4664a7507c3461a52b73] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp, totalSize=92.2 K 2023-09-29T06:14:59,813 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:59,813 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. files: [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/69a36a3b3c36467caa6890191231b396, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/9cfe148a36c24a208a3be510cd09596d, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/e761274211cc4664a7507c3461a52b73] 2023-09-29T06:14:59,813 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35056 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:14:59,813 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 18784ff25b183a4d3733801ace5c9d09/B is initiating minor compaction (all files) 2023-09-29T06:14:59,813 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 18784ff25b183a4d3733801ace5c9d09/B in TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:59,813 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/507fb535c9c343a7bc119ca6ee986318, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/3db2139efb4e480e94fb3fa12097357e, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/3aa93e87805d40dd9fcc11d990583ebe] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp, totalSize=34.2 K 2023-09-29T06:14:59,814 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 69a36a3b3c36467caa6890191231b396, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1695968095282 2023-09-29T06:14:59,814 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 507fb535c9c343a7bc119ca6ee986318, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1695968095282 2023-09-29T06:14:59,814 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 9cfe148a36c24a208a3be510cd09596d, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1695968096413 2023-09-29T06:14:59,814 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 3db2139efb4e480e94fb3fa12097357e, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1695968096413 2023-09-29T06:14:59,814 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting e761274211cc4664a7507c3461a52b73, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1695968098611 2023-09-29T06:14:59,814 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 3aa93e87805d40dd9fcc11d990583ebe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1695968098611 2023-09-29T06:14:59,819 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=18784ff25b183a4d3733801ace5c9d09] 2023-09-29T06:14:59,820 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18784ff25b183a4d3733801ace5c9d09#B#compaction#415 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:14:59,821 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/ea02f619bfde4db3bcf6206dad74b70c is 50, key is test_row_0/B:col10/1695968098611/Put/seqid=0 2023-09-29T06:14:59,823 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2023092943167d1c524248d49938de510057e34d_18784ff25b183a4d3733801ace5c9d09 store=[table=TestAcidGuarantees family=A region=18784ff25b183a4d3733801ace5c9d09] 2023-09-29T06:14:59,824 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2023092943167d1c524248d49938de510057e34d_18784ff25b183a4d3733801ace5c9d09, store=[table=TestAcidGuarantees family=A region=18784ff25b183a4d3733801ace5c9d09] 2023-09-29T06:14:59,824 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023092943167d1c524248d49938de510057e34d_18784ff25b183a4d3733801ace5c9d09 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=18784ff25b183a4d3733801ace5c9d09] 2023-09-29T06:14:59,829 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/ea02f619bfde4db3bcf6206dad74b70c as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/ea02f619bfde4db3bcf6206dad74b70c 2023-09-29T06:14:59,832 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18784ff25b183a4d3733801ace5c9d09#A#compaction#414 average throughput is 2.04 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:14:59,832 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/06db106b345c49e5b6f8f318e110c182 is 175, key is test_row_0/A:col10/1695968098611/Put/seqid=0 2023-09-29T06:14:59,834 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 18784ff25b183a4d3733801ace5c9d09/B of 18784ff25b183a4d3733801ace5c9d09 into ea02f619bfde4db3bcf6206dad74b70c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:14:59,834 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 18784ff25b183a4d3733801ace5c9d09: 2023-09-29T06:14:59,834 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09., storeName=18784ff25b183a4d3733801ace5c9d09/B, priority=13, startTime=1695968099807; duration=0sec 2023-09-29T06:14:59,834 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:14:59,834 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18784ff25b183a4d3733801ace5c9d09:B 2023-09-29T06:14:59,834 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:14:59,835 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35056 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:14:59,835 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 18784ff25b183a4d3733801ace5c9d09/C is initiating minor compaction (all files) 2023-09-29T06:14:59,835 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 18784ff25b183a4d3733801ace5c9d09/C in TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:59,835 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/f9bb3f8f09624606bd341f1cd39d918f, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/82a1c1396ef84ec082abffec18a157e6, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/8e06210a534441c59a1c5f9f78138fb4] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp, totalSize=34.2 K 2023-09-29T06:14:59,835 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting f9bb3f8f09624606bd341f1cd39d918f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1695968095282 2023-09-29T06:14:59,836 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 82a1c1396ef84ec082abffec18a157e6, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1695968096413 2023-09-29T06:14:59,836 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 8e06210a534441c59a1c5f9f78138fb4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1695968098611 2023-09-29T06:14:59,852 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/06db106b345c49e5b6f8f318e110c182 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/06db106b345c49e5b6f8f318e110c182 2023-09-29T06:14:59,853 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18784ff25b183a4d3733801ace5c9d09#C#compaction#416 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:14:59,853 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/c15572cfed694b71931fd32a8a372f99 is 50, key is test_row_0/C:col10/1695968098611/Put/seqid=0 2023-09-29T06:14:59,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2023-09-29T06:14:59,857 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 18784ff25b183a4d3733801ace5c9d09/A of 18784ff25b183a4d3733801ace5c9d09 into 06db106b345c49e5b6f8f318e110c182(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:14:59,857 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 18784ff25b183a4d3733801ace5c9d09: 2023-09-29T06:14:59,857 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09., storeName=18784ff25b183a4d3733801ace5c9d09/A, priority=13, startTime=1695968099807; duration=0sec 2023-09-29T06:14:59,857 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:59,857 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18784ff25b183a4d3733801ace5c9d09:A 2023-09-29T06:14:59,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:14:59,860 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2819): Flushing 18784ff25b183a4d3733801ace5c9d09 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2023-09-29T06:14:59,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 18784ff25b183a4d3733801ace5c9d09, store=A 2023-09-29T06:14:59,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:59,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 18784ff25b183a4d3733801ace5c9d09, store=B 2023-09-29T06:14:59,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:59,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 18784ff25b183a4d3733801ace5c9d09, store=C 2023-09-29T06:14:59,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:14:59,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929280578ac346a4caeb9cab451d4dc58d6_18784ff25b183a4d3733801ace5c9d09 is 50, key is test_row_0/A:col10/1695968098937/Put/seqid=0 2023-09-29T06:14:59,868 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/c15572cfed694b71931fd32a8a372f99 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/c15572cfed694b71931fd32a8a372f99 2023-09-29T06:14:59,872 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 18784ff25b183a4d3733801ace5c9d09/C of 18784ff25b183a4d3733801ace5c9d09 into c15572cfed694b71931fd32a8a372f99(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:14:59,873 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 18784ff25b183a4d3733801ace5c9d09: 2023-09-29T06:14:59,873 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09., storeName=18784ff25b183a4d3733801ace5c9d09/C, priority=13, startTime=1695968099807; duration=0sec 2023-09-29T06:14:59,873 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:14:59,873 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18784ff25b183a4d3733801ace5c9d09:C 2023-09-29T06:14:59,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=121 2023-09-29T06:15:00,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:15:00,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. as already flushing 2023-09-29T06:15:00,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:00,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968160162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:00,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:00,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968160162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:00,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:00,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968160266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:00,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:00,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968160266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:00,273 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929280578ac346a4caeb9cab451d4dc58d6_18784ff25b183a4d3733801ace5c9d09 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929280578ac346a4caeb9cab451d4dc58d6_18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:15:00,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/7736bc57ad244d26a8d200e74cc2d688, store: [table=TestAcidGuarantees family=A region=18784ff25b183a4d3733801ace5c9d09] 2023-09-29T06:15:00,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/7736bc57ad244d26a8d200e74cc2d688 is 175, key is test_row_0/A:col10/1695968098937/Put/seqid=0 2023-09-29T06:15:00,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:00,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968160469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:00,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:00,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968160470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:00,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=121 2023-09-29T06:15:00,679 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=326, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/7736bc57ad244d26a8d200e74cc2d688 2023-09-29T06:15:00,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/3bddca7ea0c74a1baf6c2976e4942a5f is 50, key is test_row_0/B:col10/1695968098937/Put/seqid=0 2023-09-29T06:15:00,691 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/3bddca7ea0c74a1baf6c2976e4942a5f 2023-09-29T06:15:00,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/3cc70a8694024ca7b00b9f44d4e1bb9e is 50, key is test_row_0/C:col10/1695968098937/Put/seqid=0 2023-09-29T06:15:00,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:00,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968160770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:00,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:00,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968160774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:01,056 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2307): Moving the files [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/3f28254d3ffb4eeb807f5265d15371f3, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/af6e6836c282475e8c747a7fadca8f86, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/2a4cdacace1e40a0b31b097bc59912e1, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/98953aecc029445abaf1714cc2bea07d, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/48246e66a92344cea05a1e41a8d99d8c, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/aa06083636b14f98a45965fd9c80d099, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/cf3aa8ca473b4285b872a1478f8135d0, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/867c273436c94ea1af624e3530032498, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/1252f9dcf8f24e2b9714a79de5eeefac, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/92d35f6e6a37480d812984fd295b38c7, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/12148e3f59fb44638107056a104117e5, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/82b85c2897da4d5e814473b2ae66707f, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/2481f9a016f14680bec6df08dd11ec30, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/737a22f28f244d1995a4b767e446ba73, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/3cf15b82c0be486d9b19af4be4db3ccb, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/6f469fa0c8f54a1bb02b9fea054b529e, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/51548e1b977a48a7a2c27b81d8fafaf9, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/b0ea1640bb584e5a99123c59b3cd7cb4, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/46ea766e2a214216bde59ca60c651b77, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/69a36a3b3c36467caa6890191231b396, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/9cfe148a36c24a208a3be510cd09596d, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/e761274211cc4664a7507c3461a52b73] to archive 2023-09-29T06:15:01,058 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2023-09-29T06:15:01,059 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/3f28254d3ffb4eeb807f5265d15371f3 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/3f28254d3ffb4eeb807f5265d15371f3 2023-09-29T06:15:01,060 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/af6e6836c282475e8c747a7fadca8f86 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/af6e6836c282475e8c747a7fadca8f86 2023-09-29T06:15:01,062 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/2a4cdacace1e40a0b31b097bc59912e1 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/2a4cdacace1e40a0b31b097bc59912e1 2023-09-29T06:15:01,063 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/98953aecc029445abaf1714cc2bea07d to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/98953aecc029445abaf1714cc2bea07d 2023-09-29T06:15:01,064 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/48246e66a92344cea05a1e41a8d99d8c to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/48246e66a92344cea05a1e41a8d99d8c 2023-09-29T06:15:01,065 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/aa06083636b14f98a45965fd9c80d099 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/aa06083636b14f98a45965fd9c80d099 2023-09-29T06:15:01,067 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/cf3aa8ca473b4285b872a1478f8135d0 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/cf3aa8ca473b4285b872a1478f8135d0 2023-09-29T06:15:01,068 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/867c273436c94ea1af624e3530032498 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/867c273436c94ea1af624e3530032498 2023-09-29T06:15:01,069 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/1252f9dcf8f24e2b9714a79de5eeefac to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/1252f9dcf8f24e2b9714a79de5eeefac 2023-09-29T06:15:01,071 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/92d35f6e6a37480d812984fd295b38c7 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/92d35f6e6a37480d812984fd295b38c7 2023-09-29T06:15:01,073 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/12148e3f59fb44638107056a104117e5 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/12148e3f59fb44638107056a104117e5 2023-09-29T06:15:01,074 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/82b85c2897da4d5e814473b2ae66707f to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/82b85c2897da4d5e814473b2ae66707f 2023-09-29T06:15:01,075 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/2481f9a016f14680bec6df08dd11ec30 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/2481f9a016f14680bec6df08dd11ec30 2023-09-29T06:15:01,077 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/737a22f28f244d1995a4b767e446ba73 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/737a22f28f244d1995a4b767e446ba73 2023-09-29T06:15:01,078 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/3cf15b82c0be486d9b19af4be4db3ccb to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/3cf15b82c0be486d9b19af4be4db3ccb 2023-09-29T06:15:01,079 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/6f469fa0c8f54a1bb02b9fea054b529e to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/6f469fa0c8f54a1bb02b9fea054b529e 2023-09-29T06:15:01,080 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/51548e1b977a48a7a2c27b81d8fafaf9 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/51548e1b977a48a7a2c27b81d8fafaf9 2023-09-29T06:15:01,081 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/b0ea1640bb584e5a99123c59b3cd7cb4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/b0ea1640bb584e5a99123c59b3cd7cb4 2023-09-29T06:15:01,082 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/46ea766e2a214216bde59ca60c651b77 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/46ea766e2a214216bde59ca60c651b77 2023-09-29T06:15:01,083 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/69a36a3b3c36467caa6890191231b396 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/69a36a3b3c36467caa6890191231b396 2023-09-29T06:15:01,084 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/9cfe148a36c24a208a3be510cd09596d to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/9cfe148a36c24a208a3be510cd09596d 2023-09-29T06:15:01,085 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/e761274211cc4664a7507c3461a52b73 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/e761274211cc4664a7507c3461a52b73 2023-09-29T06:15:01,090 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2307): Moving the files [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/5300f1bc83354dd9ae4085650173502b, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/c0f5feff7c904ea7b1862ba1ba033ff6, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/18b333da0f2447339da424488cb9f37b, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/8486c8e6cecf46bbb0a38dbd766f7742, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/f785dddbc0d64b9e81a45519454313e5, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/b01705fc4c7543f1ba76de60023bc591, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/0c0fa39865e24dc5850f6a17de895131, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/b10dbcadcd1b44e0ba55fc38e4f123e5, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/ccbf73ca14994422aa38aca82d258a93, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/29bc70f1d05e4dc39efc57e1f9afd723, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/96c400958bf2429196354e5a2c8db7da, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/8ff24e9e4b294023b0bf89da2db935a8, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/7dd2ac36cfb5405a835e9a1f179e6d11, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/37cb5bf2eff24a6ea73c040e115bafe1, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/748fcdd70cd2481d92a381b18f81e7e9, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/2e23c0d78c914681ab62094626265e44, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/ffd1798c93d742969dbce76cc498efbf, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/166878c24b93473d9a565df336c4a7e8, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/507fb535c9c343a7bc119ca6ee986318, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/39fc0dd678c64789bfade735e6f6818f, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/3db2139efb4e480e94fb3fa12097357e, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/3aa93e87805d40dd9fcc11d990583ebe] to archive 2023-09-29T06:15:01,091 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2023-09-29T06:15:01,093 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/5300f1bc83354dd9ae4085650173502b to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/5300f1bc83354dd9ae4085650173502b 2023-09-29T06:15:01,094 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/c0f5feff7c904ea7b1862ba1ba033ff6 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/c0f5feff7c904ea7b1862ba1ba033ff6 2023-09-29T06:15:01,096 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/18b333da0f2447339da424488cb9f37b to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/18b333da0f2447339da424488cb9f37b 2023-09-29T06:15:01,097 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/8486c8e6cecf46bbb0a38dbd766f7742 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/8486c8e6cecf46bbb0a38dbd766f7742 2023-09-29T06:15:01,098 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/f785dddbc0d64b9e81a45519454313e5 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/f785dddbc0d64b9e81a45519454313e5 2023-09-29T06:15:01,099 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/b01705fc4c7543f1ba76de60023bc591 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/b01705fc4c7543f1ba76de60023bc591 2023-09-29T06:15:01,100 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/0c0fa39865e24dc5850f6a17de895131 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/0c0fa39865e24dc5850f6a17de895131 2023-09-29T06:15:01,101 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/b10dbcadcd1b44e0ba55fc38e4f123e5 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/b10dbcadcd1b44e0ba55fc38e4f123e5 2023-09-29T06:15:01,103 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/3cc70a8694024ca7b00b9f44d4e1bb9e 2023-09-29T06:15:01,103 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/ccbf73ca14994422aa38aca82d258a93 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/ccbf73ca14994422aa38aca82d258a93 2023-09-29T06:15:01,104 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/29bc70f1d05e4dc39efc57e1f9afd723 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/29bc70f1d05e4dc39efc57e1f9afd723 2023-09-29T06:15:01,106 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/96c400958bf2429196354e5a2c8db7da to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/96c400958bf2429196354e5a2c8db7da 2023-09-29T06:15:01,107 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/8ff24e9e4b294023b0bf89da2db935a8 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/8ff24e9e4b294023b0bf89da2db935a8 2023-09-29T06:15:01,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/7736bc57ad244d26a8d200e74cc2d688 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/7736bc57ad244d26a8d200e74cc2d688 2023-09-29T06:15:01,108 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/7dd2ac36cfb5405a835e9a1f179e6d11 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/7dd2ac36cfb5405a835e9a1f179e6d11 2023-09-29T06:15:01,110 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/37cb5bf2eff24a6ea73c040e115bafe1 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/37cb5bf2eff24a6ea73c040e115bafe1 2023-09-29T06:15:01,111 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/748fcdd70cd2481d92a381b18f81e7e9 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/748fcdd70cd2481d92a381b18f81e7e9 2023-09-29T06:15:01,115 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/2e23c0d78c914681ab62094626265e44 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/2e23c0d78c914681ab62094626265e44 2023-09-29T06:15:01,116 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/ffd1798c93d742969dbce76cc498efbf to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/ffd1798c93d742969dbce76cc498efbf 2023-09-29T06:15:01,118 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/166878c24b93473d9a565df336c4a7e8 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/166878c24b93473d9a565df336c4a7e8 2023-09-29T06:15:01,119 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/507fb535c9c343a7bc119ca6ee986318 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/507fb535c9c343a7bc119ca6ee986318 2023-09-29T06:15:01,120 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/39fc0dd678c64789bfade735e6f6818f to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/39fc0dd678c64789bfade735e6f6818f 2023-09-29T06:15:01,121 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/3db2139efb4e480e94fb3fa12097357e to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/3db2139efb4e480e94fb3fa12097357e 2023-09-29T06:15:01,122 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/7736bc57ad244d26a8d200e74cc2d688, entries=100, sequenceid=326, filesize=22.0 K 2023-09-29T06:15:01,123 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/3aa93e87805d40dd9fcc11d990583ebe to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/3aa93e87805d40dd9fcc11d990583ebe 2023-09-29T06:15:01,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/3bddca7ea0c74a1baf6c2976e4942a5f as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/3bddca7ea0c74a1baf6c2976e4942a5f 2023-09-29T06:15:01,129 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2307): Moving the files [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/1d141bd1a12241b3abbca77fd2290c24, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/1f9b0591292643fca3eafe3be22a1db6, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/63f37e3c1000450fa80c97636db2612e, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/2e67ec3f817d4407b59a53f2fb2187c0, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/284043b7fae9498bae42c1f08b73c64d, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/96e93103a4854ba59af31f6008e44357, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/83f70e818439417bbd5dff786adc0235, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/92ba6f4e97ac4325918303ece8909753, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/5ba41f4a26a54cdd9c9178eeb2624ee3, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/53df647d2c624fdc9b487f44b43d4d02, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/d61b5d8ce4df43f2996808a49ab091af, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/18554b3c86bf428b9dc43f5b170d1fb3, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/33fcd74ed59047f9a803a530a4893f74, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/308b460ef3d84139b3edbd06ffe4bf18, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/b9a8a7b328dd4434a09c4ce18d34faae, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/7e79cbd361a54916b8d7908d537a1d58, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/2c4316ff16874f8ebf84001486298493, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/6f689318790145a49b49c40761ee9175, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/f9bb3f8f09624606bd341f1cd39d918f, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/a2a2c21eb9da475a8adc3f1306422527, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/82a1c1396ef84ec082abffec18a157e6, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/8e06210a534441c59a1c5f9f78138fb4] to archive 2023-09-29T06:15:01,130 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2023-09-29T06:15:01,130 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/3bddca7ea0c74a1baf6c2976e4942a5f, entries=100, sequenceid=326, filesize=9.6 K 2023-09-29T06:15:01,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/3cc70a8694024ca7b00b9f44d4e1bb9e as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/3cc70a8694024ca7b00b9f44d4e1bb9e 2023-09-29T06:15:01,131 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/1d141bd1a12241b3abbca77fd2290c24 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/1d141bd1a12241b3abbca77fd2290c24 2023-09-29T06:15:01,133 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/1f9b0591292643fca3eafe3be22a1db6 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/1f9b0591292643fca3eafe3be22a1db6 2023-09-29T06:15:01,135 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/63f37e3c1000450fa80c97636db2612e to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/63f37e3c1000450fa80c97636db2612e 2023-09-29T06:15:01,136 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/2e67ec3f817d4407b59a53f2fb2187c0 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/2e67ec3f817d4407b59a53f2fb2187c0 2023-09-29T06:15:01,137 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/284043b7fae9498bae42c1f08b73c64d to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/284043b7fae9498bae42c1f08b73c64d 2023-09-29T06:15:01,138 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/96e93103a4854ba59af31f6008e44357 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/96e93103a4854ba59af31f6008e44357 2023-09-29T06:15:01,139 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/3cc70a8694024ca7b00b9f44d4e1bb9e, entries=100, sequenceid=326, filesize=9.6 K 2023-09-29T06:15:01,140 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3022): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=187.85 KB/192360 for 18784ff25b183a4d3733801ace5c9d09 in 1280ms, sequenceid=326, compaction requested=false 2023-09-29T06:15:01,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2520): Flush status journal for 18784ff25b183a4d3733801ace5c9d09: 2023-09-29T06:15:01,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:15:01,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=122 2023-09-29T06:15:01,140 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/83f70e818439417bbd5dff786adc0235 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/83f70e818439417bbd5dff786adc0235 2023-09-29T06:15:01,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=122 2023-09-29T06:15:01,141 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/92ba6f4e97ac4325918303ece8909753 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/92ba6f4e97ac4325918303ece8909753 2023-09-29T06:15:01,143 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/5ba41f4a26a54cdd9c9178eeb2624ee3 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/5ba41f4a26a54cdd9c9178eeb2624ee3 2023-09-29T06:15:01,143 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=122, resume processing ppid=121 2023-09-29T06:15:01,143 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=122, ppid=121, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7440 sec 2023-09-29T06:15:01,144 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/53df647d2c624fdc9b487f44b43d4d02 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/53df647d2c624fdc9b487f44b43d4d02 2023-09-29T06:15:01,144 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=121, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.7490 sec 2023-09-29T06:15:01,146 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/d61b5d8ce4df43f2996808a49ab091af to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/d61b5d8ce4df43f2996808a49ab091af 2023-09-29T06:15:01,147 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/18554b3c86bf428b9dc43f5b170d1fb3 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/18554b3c86bf428b9dc43f5b170d1fb3 2023-09-29T06:15:01,148 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/33fcd74ed59047f9a803a530a4893f74 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/33fcd74ed59047f9a803a530a4893f74 2023-09-29T06:15:01,150 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/308b460ef3d84139b3edbd06ffe4bf18 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/308b460ef3d84139b3edbd06ffe4bf18 2023-09-29T06:15:01,152 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/b9a8a7b328dd4434a09c4ce18d34faae to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/b9a8a7b328dd4434a09c4ce18d34faae 2023-09-29T06:15:01,153 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/7e79cbd361a54916b8d7908d537a1d58 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/7e79cbd361a54916b8d7908d537a1d58 2023-09-29T06:15:01,155 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/2c4316ff16874f8ebf84001486298493 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/2c4316ff16874f8ebf84001486298493 2023-09-29T06:15:01,157 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/6f689318790145a49b49c40761ee9175 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/6f689318790145a49b49c40761ee9175 2023-09-29T06:15:01,158 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/f9bb3f8f09624606bd341f1cd39d918f to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/f9bb3f8f09624606bd341f1cd39d918f 2023-09-29T06:15:01,159 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/a2a2c21eb9da475a8adc3f1306422527 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/a2a2c21eb9da475a8adc3f1306422527 2023-09-29T06:15:01,161 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/82a1c1396ef84ec082abffec18a157e6 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/82a1c1396ef84ec082abffec18a157e6 2023-09-29T06:15:01,161 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/8e06210a534441c59a1c5f9f78138fb4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/8e06210a534441c59a1c5f9f78138fb4 2023-09-29T06:15:01,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:15:01,278 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 18784ff25b183a4d3733801ace5c9d09 3/3 column families, dataSize=194.56 KB heapSize=510.52 KB 2023-09-29T06:15:01,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 18784ff25b183a4d3733801ace5c9d09, store=A 2023-09-29T06:15:01,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:01,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 18784ff25b183a4d3733801ace5c9d09, store=B 2023-09-29T06:15:01,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:01,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 18784ff25b183a4d3733801ace5c9d09, store=C 2023-09-29T06:15:01,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:01,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:01,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968161278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:01,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:01,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968161280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:01,283 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929f5bd456db2214fb999fbdd539304f45c_18784ff25b183a4d3733801ace5c9d09 is 50, key is test_row_0/A:col10/1695968100161/Put/seqid=0 2023-09-29T06:15:01,291 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929f5bd456db2214fb999fbdd539304f45c_18784ff25b183a4d3733801ace5c9d09 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929f5bd456db2214fb999fbdd539304f45c_18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:15:01,291 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/9bd8854cfd6f4d79b512a50dcb94a1d4, store: [table=TestAcidGuarantees family=A region=18784ff25b183a4d3733801ace5c9d09] 2023-09-29T06:15:01,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/9bd8854cfd6f4d79b512a50dcb94a1d4 is 175, key is test_row_0/A:col10/1695968100161/Put/seqid=0 2023-09-29T06:15:01,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:01,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968161381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:01,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:01,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968161382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:01,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=121 2023-09-29T06:15:01,500 INFO [Thread-2144 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 121 completed 2023-09-29T06:15:01,502 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:15:01,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2023-09-29T06:15:01,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=123 2023-09-29T06:15:01,504 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:15:01,505 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:15:01,505 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:15:01,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:01,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968161583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:01,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:01,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968161585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:01,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=123 2023-09-29T06:15:01,614 DEBUG [Thread-2153 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x6b9d6aa4 to 127.0.0.1:49344 2023-09-29T06:15:01,615 DEBUG [Thread-2153 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:15:01,615 DEBUG [Thread-2145 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x3cf858b9 to 127.0.0.1:49344 2023-09-29T06:15:01,615 DEBUG [Thread-2145 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:15:01,615 DEBUG [Thread-2147 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x581a0a23 to 127.0.0.1:49344 2023-09-29T06:15:01,615 DEBUG [Thread-2147 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:15:01,618 DEBUG [Thread-2149 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x3c59e028 to 127.0.0.1:49344 2023-09-29T06:15:01,618 DEBUG [Thread-2151 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x65c2b0e0 to 127.0.0.1:49344 2023-09-29T06:15:01,618 DEBUG [Thread-2149 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:15:01,618 DEBUG [Thread-2151 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:15:01,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2023-09-29T06:15:01,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:15:01,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. as already flushing 2023-09-29T06:15:01,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:15:01,658 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(53): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:01,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:01,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:01,697 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=359, memsize=64.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/9bd8854cfd6f4d79b512a50dcb94a1d4 2023-09-29T06:15:01,702 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/3147bda1f3cc4ef195c653f46fb3cad5 is 50, key is test_row_0/B:col10/1695968100161/Put/seqid=0 2023-09-29T06:15:01,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=123 2023-09-29T06:15:01,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2023-09-29T06:15:01,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:15:01,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. as already flushing 2023-09-29T06:15:01,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:15:01,811 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(53): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:01,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:01,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:01,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:01,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:01,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968161889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:01,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968161889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:01,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2023-09-29T06:15:01,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:15:01,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. as already flushing 2023-09-29T06:15:01,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:15:01,963 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(53): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:01,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:01,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:02,106 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/3147bda1f3cc4ef195c653f46fb3cad5 2023-09-29T06:15:02,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=123 2023-09-29T06:15:02,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/6a28f2facbed426ca86d6914b3ce8296 is 50, key is test_row_0/C:col10/1695968100161/Put/seqid=0 2023-09-29T06:15:02,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2023-09-29T06:15:02,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:15:02,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. as already flushing 2023-09-29T06:15:02,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:15:02,115 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(53): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:02,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:02,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:02,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2023-09-29T06:15:02,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:15:02,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. as already flushing 2023-09-29T06:15:02,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:15:02,268 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(53): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:02,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:02,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:02,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:02,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52170 deadline: 1695968162391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:02,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:02,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:52138 deadline: 1695968162394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:02,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2023-09-29T06:15:02,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:15:02,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. as already flushing 2023-09-29T06:15:02,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:15:02,420 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(53): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:02,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:02,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:02,516 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/6a28f2facbed426ca86d6914b3ce8296 2023-09-29T06:15:02,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/9bd8854cfd6f4d79b512a50dcb94a1d4 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/9bd8854cfd6f4d79b512a50dcb94a1d4 2023-09-29T06:15:02,522 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/9bd8854cfd6f4d79b512a50dcb94a1d4, entries=200, sequenceid=359, filesize=39.0 K 2023-09-29T06:15:02,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/3147bda1f3cc4ef195c653f46fb3cad5 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/3147bda1f3cc4ef195c653f46fb3cad5 2023-09-29T06:15:02,525 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/3147bda1f3cc4ef195c653f46fb3cad5, entries=150, sequenceid=359, filesize=12.0 K 2023-09-29T06:15:02,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/6a28f2facbed426ca86d6914b3ce8296 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/6a28f2facbed426ca86d6914b3ce8296 2023-09-29T06:15:02,529 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/6a28f2facbed426ca86d6914b3ce8296, entries=150, sequenceid=359, filesize=12.0 K 2023-09-29T06:15:02,529 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~194.56 KB/199230, heapSize ~510.47 KB/522720, currentSize=6.71 KB/6870 for 18784ff25b183a4d3733801ace5c9d09 in 1251ms, sequenceid=359, compaction requested=true 2023-09-29T06:15:02,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 18784ff25b183a4d3733801ace5c9d09: 2023-09-29T06:15:02,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18784ff25b183a4d3733801ace5c9d09:A, priority=-2147483648, current under compaction store size is 1 2023-09-29T06:15:02,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:02,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18784ff25b183a4d3733801ace5c9d09:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:15:02,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:02,529 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:15:02,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18784ff25b183a4d3733801ace5c9d09:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:15:02,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:15:02,530 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:15:02,530 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94464 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:15:02,530 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 18784ff25b183a4d3733801ace5c9d09/A is initiating minor compaction (all files) 2023-09-29T06:15:02,530 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35158 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:15:02,530 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 18784ff25b183a4d3733801ace5c9d09/A in TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:15:02,530 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 18784ff25b183a4d3733801ace5c9d09/B is initiating minor compaction (all files) 2023-09-29T06:15:02,530 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/06db106b345c49e5b6f8f318e110c182, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/7736bc57ad244d26a8d200e74cc2d688, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/9bd8854cfd6f4d79b512a50dcb94a1d4] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp, totalSize=92.3 K 2023-09-29T06:15:02,530 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 18784ff25b183a4d3733801ace5c9d09/B in TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:15:02,531 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:15:02,531 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/ea02f619bfde4db3bcf6206dad74b70c, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/3bddca7ea0c74a1baf6c2976e4942a5f, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/3147bda1f3cc4ef195c653f46fb3cad5] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp, totalSize=34.3 K 2023-09-29T06:15:02,531 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. files: [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/06db106b345c49e5b6f8f318e110c182, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/7736bc57ad244d26a8d200e74cc2d688, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/9bd8854cfd6f4d79b512a50dcb94a1d4] 2023-09-29T06:15:02,531 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 06db106b345c49e5b6f8f318e110c182, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1695968098611 2023-09-29T06:15:02,531 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting ea02f619bfde4db3bcf6206dad74b70c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1695968098611 2023-09-29T06:15:02,531 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 7736bc57ad244d26a8d200e74cc2d688, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1695968098937 2023-09-29T06:15:02,531 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 3bddca7ea0c74a1baf6c2976e4942a5f, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1695968098937 2023-09-29T06:15:02,531 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 9bd8854cfd6f4d79b512a50dcb94a1d4, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1695968100141 2023-09-29T06:15:02,532 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 3147bda1f3cc4ef195c653f46fb3cad5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1695968100151 2023-09-29T06:15:02,536 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=18784ff25b183a4d3733801ace5c9d09] 2023-09-29T06:15:02,536 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18784ff25b183a4d3733801ace5c9d09#B#compaction#423 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:02,537 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/1a76e6a0aa6d4d7e955e19a1343d5a3c is 50, key is test_row_0/B:col10/1695968100161/Put/seqid=0 2023-09-29T06:15:02,537 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202309299aa5316f810c4056b180bf67c9010ff0_18784ff25b183a4d3733801ace5c9d09 store=[table=TestAcidGuarantees family=A region=18784ff25b183a4d3733801ace5c9d09] 2023-09-29T06:15:02,541 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202309299aa5316f810c4056b180bf67c9010ff0_18784ff25b183a4d3733801ace5c9d09, store=[table=TestAcidGuarantees family=A region=18784ff25b183a4d3733801ace5c9d09] 2023-09-29T06:15:02,541 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309299aa5316f810c4056b180bf67c9010ff0_18784ff25b183a4d3733801ace5c9d09 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=18784ff25b183a4d3733801ace5c9d09] 2023-09-29T06:15:02,548 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/1a76e6a0aa6d4d7e955e19a1343d5a3c as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/1a76e6a0aa6d4d7e955e19a1343d5a3c 2023-09-29T06:15:02,551 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 18784ff25b183a4d3733801ace5c9d09/B of 18784ff25b183a4d3733801ace5c9d09 into 1a76e6a0aa6d4d7e955e19a1343d5a3c(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:02,551 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 18784ff25b183a4d3733801ace5c9d09: 2023-09-29T06:15:02,551 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09., storeName=18784ff25b183a4d3733801ace5c9d09/B, priority=13, startTime=1695968102529; duration=0sec 2023-09-29T06:15:02,551 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:15:02,551 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18784ff25b183a4d3733801ace5c9d09:B 2023-09-29T06:15:02,551 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:15:02,552 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35158 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:15:02,552 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 18784ff25b183a4d3733801ace5c9d09/C is initiating minor compaction (all files) 2023-09-29T06:15:02,552 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 18784ff25b183a4d3733801ace5c9d09/C in TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:15:02,552 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/c15572cfed694b71931fd32a8a372f99, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/3cc70a8694024ca7b00b9f44d4e1bb9e, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/6a28f2facbed426ca86d6914b3ce8296] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp, totalSize=34.3 K 2023-09-29T06:15:02,552 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting c15572cfed694b71931fd32a8a372f99, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1695968098611 2023-09-29T06:15:02,552 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 3cc70a8694024ca7b00b9f44d4e1bb9e, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1695968098937 2023-09-29T06:15:02,553 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 6a28f2facbed426ca86d6914b3ce8296, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1695968100151 2023-09-29T06:15:02,557 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18784ff25b183a4d3733801ace5c9d09#C#compaction#425 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:02,557 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/a649c52a35b548a189972f115551efe3 is 50, key is test_row_0/C:col10/1695968100161/Put/seqid=0 2023-09-29T06:15:02,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2023-09-29T06:15:02,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:15:02,572 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2819): Flushing 18784ff25b183a4d3733801ace5c9d09 3/3 column families, dataSize=6.71 KB heapSize=18.33 KB 2023-09-29T06:15:02,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 18784ff25b183a4d3733801ace5c9d09, store=A 2023-09-29T06:15:02,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:02,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 18784ff25b183a4d3733801ace5c9d09, store=B 2023-09-29T06:15:02,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:02,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 18784ff25b183a4d3733801ace5c9d09, store=C 2023-09-29T06:15:02,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:02,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023092998e6930a24df491d8fa5623d6351e7e8_18784ff25b183a4d3733801ace5c9d09 is 50, key is test_row_0/A:col10/1695968101279/Put/seqid=0 2023-09-29T06:15:02,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=123 2023-09-29T06:15:02,948 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18784ff25b183a4d3733801ace5c9d09#A#compaction#424 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:02,949 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/cfc9b86d4f04459cb34b8d9f01d3f013 is 175, key is test_row_0/A:col10/1695968100161/Put/seqid=0 2023-09-29T06:15:02,976 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/a649c52a35b548a189972f115551efe3 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/a649c52a35b548a189972f115551efe3 2023-09-29T06:15:02,980 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 18784ff25b183a4d3733801ace5c9d09/C of 18784ff25b183a4d3733801ace5c9d09 into a649c52a35b548a189972f115551efe3(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:02,980 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 18784ff25b183a4d3733801ace5c9d09: 2023-09-29T06:15:02,980 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09., storeName=18784ff25b183a4d3733801ace5c9d09/C, priority=13, startTime=1695968102529; duration=0sec 2023-09-29T06:15:02,980 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:02,980 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18784ff25b183a4d3733801ace5c9d09:C 2023-09-29T06:15:02,981 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023092998e6930a24df491d8fa5623d6351e7e8_18784ff25b183a4d3733801ace5c9d09 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023092998e6930a24df491d8fa5623d6351e7e8_18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:15:02,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/3d01c5b67e4a4d2fb4bc33ef5747808a, store: [table=TestAcidGuarantees family=A region=18784ff25b183a4d3733801ace5c9d09] 2023-09-29T06:15:02,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/3d01c5b67e4a4d2fb4bc33ef5747808a is 175, key is test_row_0/A:col10/1695968101279/Put/seqid=0 2023-09-29T06:15:03,355 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/cfc9b86d4f04459cb34b8d9f01d3f013 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/cfc9b86d4f04459cb34b8d9f01d3f013 2023-09-29T06:15:03,359 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 18784ff25b183a4d3733801ace5c9d09/A of 18784ff25b183a4d3733801ace5c9d09 into cfc9b86d4f04459cb34b8d9f01d3f013(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:03,359 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 18784ff25b183a4d3733801ace5c9d09: 2023-09-29T06:15:03,359 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09., storeName=18784ff25b183a4d3733801ace5c9d09/A, priority=13, startTime=1695968102529; duration=0sec 2023-09-29T06:15:03,359 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:03,359 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18784ff25b183a4d3733801ace5c9d09:A 2023-09-29T06:15:03,386 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=364, memsize=2.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/3d01c5b67e4a4d2fb4bc33ef5747808a 2023-09-29T06:15:03,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/96c5a14dc5c144f693532d6a5c4e6183 is 50, key is test_row_0/B:col10/1695968101279/Put/seqid=0 2023-09-29T06:15:03,399 DEBUG [Thread-2140 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x51c9bce9 to 127.0.0.1:49344 2023-09-29T06:15:03,399 DEBUG [Thread-2140 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:15:03,400 DEBUG [Thread-2142 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x444cbd8a to 127.0.0.1:49344 2023-09-29T06:15:03,400 DEBUG [Thread-2142 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:15:03,466 DEBUG [Thread-2134 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x405600e8 to 127.0.0.1:49344 2023-09-29T06:15:03,466 DEBUG [Thread-2134 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:15:03,479 DEBUG [Thread-2138 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x798e79ee to 127.0.0.1:49344 2023-09-29T06:15:03,479 DEBUG [Thread-2138 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:15:03,495 DEBUG [Thread-2136 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x158c1b4d to 127.0.0.1:49344 2023-09-29T06:15:03,495 DEBUG [Thread-2136 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:15:03,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=123 2023-09-29T06:15:03,794 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/96c5a14dc5c144f693532d6a5c4e6183 2023-09-29T06:15:03,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/2ee02ad2e9814ba5a8e7b745dd6edd21 is 50, key is test_row_0/C:col10/1695968101279/Put/seqid=0 2023-09-29T06:15:04,204 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/2ee02ad2e9814ba5a8e7b745dd6edd21 2023-09-29T06:15:04,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/3d01c5b67e4a4d2fb4bc33ef5747808a as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/3d01c5b67e4a4d2fb4bc33ef5747808a 2023-09-29T06:15:04,210 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/3d01c5b67e4a4d2fb4bc33ef5747808a, entries=50, sequenceid=364, filesize=13.5 K 2023-09-29T06:15:04,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/96c5a14dc5c144f693532d6a5c4e6183 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/96c5a14dc5c144f693532d6a5c4e6183 2023-09-29T06:15:04,213 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/96c5a14dc5c144f693532d6a5c4e6183, entries=50, sequenceid=364, filesize=7.2 K 2023-09-29T06:15:04,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/2ee02ad2e9814ba5a8e7b745dd6edd21 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/2ee02ad2e9814ba5a8e7b745dd6edd21 2023-09-29T06:15:04,217 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/2ee02ad2e9814ba5a8e7b745dd6edd21, entries=50, sequenceid=364, filesize=7.2 K 2023-09-29T06:15:04,218 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3022): Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=33.54 KB/34350 for 18784ff25b183a4d3733801ace5c9d09 in 1645ms, sequenceid=364, compaction requested=false 2023-09-29T06:15:04,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2520): Flush status journal for 18784ff25b183a4d3733801ace5c9d09: 2023-09-29T06:15:04,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:15:04,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=124 2023-09-29T06:15:04,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=124 2023-09-29T06:15:04,220 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=124, resume processing ppid=123 2023-09-29T06:15:04,220 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=124, ppid=123, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7140 sec 2023-09-29T06:15:04,221 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=123, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 2.7180 sec 2023-09-29T06:15:05,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=123 2023-09-29T06:15:05,610 INFO [Thread-2144 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 123 completed 2023-09-29T06:15:05,610 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2023-09-29T06:15:05,610 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(394): wrote 41 2023-09-29T06:15:05,610 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(394): wrote 38 2023-09-29T06:15:05,610 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2023-09-29T06:15:05,610 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(394): wrote 79 2023-09-29T06:15:05,610 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2023-09-29T06:15:05,610 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(396): Readers: 2023-09-29T06:15:05,610 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2023-09-29T06:15:05,610 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(402): scanned 2684 2023-09-29T06:15:05,610 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(403): verified 8049 rows 2023-09-29T06:15:05,610 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(402): scanned 2677 2023-09-29T06:15:05,610 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(403): verified 8028 rows 2023-09-29T06:15:05,610 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(402): scanned 2684 2023-09-29T06:15:05,611 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(403): verified 8052 rows 2023-09-29T06:15:05,611 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(402): scanned 2696 2023-09-29T06:15:05,611 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(403): verified 8085 rows 2023-09-29T06:15:05,611 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(402): scanned 2690 2023-09-29T06:15:05,611 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(403): verified 8070 rows 2023-09-29T06:15:05,611 INFO [Listener at localhost/42843 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-09-29T06:15:05,611 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x4fd4369c to 127.0.0.1:49344 2023-09-29T06:15:05,611 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:15:05,612 INFO [Listener at localhost/42843 {}] client.HBaseAdmin$18(967): Started disable of TestAcidGuarantees 2023-09-29T06:15:05,613 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$12(2664): Client=jenkins//172.31.12.81 disable TestAcidGuarantees 2023-09-29T06:15:05,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=125, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=TestAcidGuarantees 2023-09-29T06:15:05,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=125 2023-09-29T06:15:05,618 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968105617"}]},"ts":"1695968105617"} 2023-09-29T06:15:05,619 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2023-09-29T06:15:05,621 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(293): Set TestAcidGuarantees to state=DISABLING 2023-09-29T06:15:05,621 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=18784ff25b183a4d3733801ace5c9d09, UNASSIGN}] 2023-09-29T06:15:05,622 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=126, ppid=125, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=18784ff25b183a4d3733801ace5c9d09, UNASSIGN 2023-09-29T06:15:05,623 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=126 updating hbase:meta row=18784ff25b183a4d3733801ace5c9d09, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:05,624 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=127, ppid=126, state=RUNNABLE; CloseRegionProcedure 18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138}] 2023-09-29T06:15:05,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=125 2023-09-29T06:15:05,775 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] handler.UnassignRegionHandler(119): Close 18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:15:05,775 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] regionserver.HRegion(1668): Closing 18784ff25b183a4d3733801ace5c9d09, disabling compactions & flushes 2023-09-29T06:15:05,776 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] regionserver.HRegion(1690): Closing region TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:15:05,776 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] regionserver.HRegion(1711): Waiting without time limit for close lock on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:15:05,776 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] regionserver.HRegion(1778): Acquired close lock on TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. after waiting 0 ms 2023-09-29T06:15:05,776 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] regionserver.HRegion(1788): Updates disabled for region TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:15:05,776 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] regionserver.HRegion(2819): Flushing 18784ff25b183a4d3733801ace5c9d09 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2023-09-29T06:15:05,776 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 18784ff25b183a4d3733801ace5c9d09, store=A 2023-09-29T06:15:05,776 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:05,776 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 18784ff25b183a4d3733801ace5c9d09, store=B 2023-09-29T06:15:05,776 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:05,776 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 18784ff25b183a4d3733801ace5c9d09, store=C 2023-09-29T06:15:05,776 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:05,780 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929de594f84e46b44a196dea08fa0522524_18784ff25b183a4d3733801ace5c9d09 is 50, key is test_row_0/A:col10/1695968103494/Put/seqid=0 2023-09-29T06:15:05,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=125 2023-09-29T06:15:06,187 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929de594f84e46b44a196dea08fa0522524_18784ff25b183a4d3733801ace5c9d09 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929de594f84e46b44a196dea08fa0522524_18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:15:06,188 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/fdc710a8260d4648848b4137601bbfb3, store: [table=TestAcidGuarantees family=A region=18784ff25b183a4d3733801ace5c9d09] 2023-09-29T06:15:06,189 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/fdc710a8260d4648848b4137601bbfb3 is 175, key is test_row_0/A:col10/1695968103494/Put/seqid=0 2023-09-29T06:15:06,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=125 2023-09-29T06:15:06,592 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=374, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/fdc710a8260d4648848b4137601bbfb3 2023-09-29T06:15:06,597 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/b73b978de6c049fcbb2010cb6f71bb9e is 50, key is test_row_0/B:col10/1695968103494/Put/seqid=0 2023-09-29T06:15:06,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=125 2023-09-29T06:15:07,001 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/b73b978de6c049fcbb2010cb6f71bb9e 2023-09-29T06:15:07,006 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/dc46cde53e59413da6f41b0df78e13b8 is 50, key is test_row_0/C:col10/1695968103494/Put/seqid=0 2023-09-29T06:15:07,410 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/dc46cde53e59413da6f41b0df78e13b8 2023-09-29T06:15:07,413 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/A/fdc710a8260d4648848b4137601bbfb3 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/fdc710a8260d4648848b4137601bbfb3 2023-09-29T06:15:07,416 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/fdc710a8260d4648848b4137601bbfb3, entries=100, sequenceid=374, filesize=22.0 K 2023-09-29T06:15:07,416 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/B/b73b978de6c049fcbb2010cb6f71bb9e as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/b73b978de6c049fcbb2010cb6f71bb9e 2023-09-29T06:15:07,419 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/b73b978de6c049fcbb2010cb6f71bb9e, entries=100, sequenceid=374, filesize=9.6 K 2023-09-29T06:15:07,420 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/.tmp/C/dc46cde53e59413da6f41b0df78e13b8 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/dc46cde53e59413da6f41b0df78e13b8 2023-09-29T06:15:07,423 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/dc46cde53e59413da6f41b0df78e13b8, entries=100, sequenceid=374, filesize=9.6 K 2023-09-29T06:15:07,423 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] regionserver.HRegion(3022): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 18784ff25b183a4d3733801ace5c9d09 in 1647ms, sequenceid=374, compaction requested=true 2023-09-29T06:15:07,424 DEBUG [StoreCloser-TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/06db106b345c49e5b6f8f318e110c182, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/7736bc57ad244d26a8d200e74cc2d688, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/9bd8854cfd6f4d79b512a50dcb94a1d4] to archive 2023-09-29T06:15:07,424 DEBUG [StoreCloser-TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2023-09-29T06:15:07,425 DEBUG [StoreCloser-TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/06db106b345c49e5b6f8f318e110c182 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/06db106b345c49e5b6f8f318e110c182 2023-09-29T06:15:07,426 DEBUG [StoreCloser-TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/7736bc57ad244d26a8d200e74cc2d688 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/7736bc57ad244d26a8d200e74cc2d688 2023-09-29T06:15:07,427 DEBUG [StoreCloser-TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/9bd8854cfd6f4d79b512a50dcb94a1d4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/9bd8854cfd6f4d79b512a50dcb94a1d4 2023-09-29T06:15:07,428 DEBUG [StoreCloser-TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/ea02f619bfde4db3bcf6206dad74b70c, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/3bddca7ea0c74a1baf6c2976e4942a5f, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/3147bda1f3cc4ef195c653f46fb3cad5] to archive 2023-09-29T06:15:07,428 DEBUG [StoreCloser-TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2023-09-29T06:15:07,429 DEBUG [StoreCloser-TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/ea02f619bfde4db3bcf6206dad74b70c to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/ea02f619bfde4db3bcf6206dad74b70c 2023-09-29T06:15:07,430 DEBUG [StoreCloser-TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/3bddca7ea0c74a1baf6c2976e4942a5f to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/3bddca7ea0c74a1baf6c2976e4942a5f 2023-09-29T06:15:07,431 DEBUG [StoreCloser-TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/3147bda1f3cc4ef195c653f46fb3cad5 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/3147bda1f3cc4ef195c653f46fb3cad5 2023-09-29T06:15:07,431 DEBUG [StoreCloser-TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/c15572cfed694b71931fd32a8a372f99, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/3cc70a8694024ca7b00b9f44d4e1bb9e, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/6a28f2facbed426ca86d6914b3ce8296] to archive 2023-09-29T06:15:07,432 DEBUG [StoreCloser-TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2023-09-29T06:15:07,433 DEBUG [StoreCloser-TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/c15572cfed694b71931fd32a8a372f99 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/c15572cfed694b71931fd32a8a372f99 2023-09-29T06:15:07,434 DEBUG [StoreCloser-TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/3cc70a8694024ca7b00b9f44d4e1bb9e to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/3cc70a8694024ca7b00b9f44d4e1bb9e 2023-09-29T06:15:07,435 DEBUG [StoreCloser-TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/6a28f2facbed426ca86d6914b3ce8296 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/6a28f2facbed426ca86d6914b3ce8296 2023-09-29T06:15:07,438 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/recovered.edits/377.seqid, newMaxSeqId=377, maxSeqId=4 2023-09-29T06:15:07,439 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] regionserver.HRegion(1909): Closed TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09. 2023-09-29T06:15:07,439 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] regionserver.HRegion(1622): Region close journal for 18784ff25b183a4d3733801ace5c9d09: 2023-09-29T06:15:07,440 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=127}] handler.UnassignRegionHandler(163): Closed 18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:15:07,441 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=126 updating hbase:meta row=18784ff25b183a4d3733801ace5c9d09, regionState=CLOSED 2023-09-29T06:15:07,442 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=127, resume processing ppid=126 2023-09-29T06:15:07,442 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=127, ppid=126, state=SUCCESS; CloseRegionProcedure 18784ff25b183a4d3733801ace5c9d09, server=jenkins-hbase3.apache.org,39783,1695967978138 in 1.8170 sec 2023-09-29T06:15:07,443 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=126, resume processing ppid=125 2023-09-29T06:15:07,443 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=126, ppid=125, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=18784ff25b183a4d3733801ace5c9d09, UNASSIGN in 1.8210 sec 2023-09-29T06:15:07,444 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968107444"}]},"ts":"1695968107444"} 2023-09-29T06:15:07,444 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2023-09-29T06:15:07,457 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(305): Set TestAcidGuarantees to state=DISABLED 2023-09-29T06:15:07,458 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=125, state=SUCCESS; DisableTableProcedure table=TestAcidGuarantees in 1.8440 sec 2023-09-29T06:15:07,460 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2023-09-29T06:15:07,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=125 2023-09-29T06:15:07,721 INFO [Listener at localhost/42843 {}] client.HBaseAdmin$TableFuture(3677): Operation: DISABLE, Table Name: default:TestAcidGuarantees, procId: 125 completed 2023-09-29T06:15:07,722 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$5(2448): Client=jenkins//172.31.12.81 delete TestAcidGuarantees 2023-09-29T06:15:07,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=128, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=TestAcidGuarantees 2023-09-29T06:15:07,724 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=128, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2023-09-29T06:15:07,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=128 2023-09-29T06:15:07,724 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=128, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2023-09-29T06:15:07,728 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:15:07,730 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A, FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B, FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C, FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/recovered.edits] 2023-09-29T06:15:07,732 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/3d01c5b67e4a4d2fb4bc33ef5747808a to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/3d01c5b67e4a4d2fb4bc33ef5747808a 2023-09-29T06:15:07,733 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/cfc9b86d4f04459cb34b8d9f01d3f013 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/cfc9b86d4f04459cb34b8d9f01d3f013 2023-09-29T06:15:07,734 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/fdc710a8260d4648848b4137601bbfb3 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/A/fdc710a8260d4648848b4137601bbfb3 2023-09-29T06:15:07,736 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/1a76e6a0aa6d4d7e955e19a1343d5a3c to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/1a76e6a0aa6d4d7e955e19a1343d5a3c 2023-09-29T06:15:07,737 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/96c5a14dc5c144f693532d6a5c4e6183 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/96c5a14dc5c144f693532d6a5c4e6183 2023-09-29T06:15:07,738 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/b73b978de6c049fcbb2010cb6f71bb9e to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/B/b73b978de6c049fcbb2010cb6f71bb9e 2023-09-29T06:15:07,740 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/2ee02ad2e9814ba5a8e7b745dd6edd21 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/2ee02ad2e9814ba5a8e7b745dd6edd21 2023-09-29T06:15:07,741 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/a649c52a35b548a189972f115551efe3 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/a649c52a35b548a189972f115551efe3 2023-09-29T06:15:07,742 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/dc46cde53e59413da6f41b0df78e13b8 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/C/dc46cde53e59413da6f41b0df78e13b8 2023-09-29T06:15:07,744 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/recovered.edits/377.seqid to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09/recovered.edits/377.seqid 2023-09-29T06:15:07,744 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:15:07,744 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(293): Archived TestAcidGuarantees regions 2023-09-29T06:15:07,744 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2023-09-29T06:15:07,745 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2023-09-29T06:15:07,747 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023092915d77c8d351945f9aa91402b7827fda5_18784ff25b183a4d3733801ace5c9d09 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023092915d77c8d351945f9aa91402b7827fda5_18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:15:07,748 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023092916904332e2e64d299cb1437967f6664f_18784ff25b183a4d3733801ace5c9d09 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023092916904332e2e64d299cb1437967f6664f_18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:15:07,749 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309291d00de74d3c34ca8b0ed35becefcb780_18784ff25b183a4d3733801ace5c9d09 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309291d00de74d3c34ca8b0ed35becefcb780_18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:15:07,750 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929280578ac346a4caeb9cab451d4dc58d6_18784ff25b183a4d3733801ace5c9d09 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929280578ac346a4caeb9cab451d4dc58d6_18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:15:07,751 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309294e7fffac01e04ee6947f444bc21f0af0_18784ff25b183a4d3733801ace5c9d09 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309294e7fffac01e04ee6947f444bc21f0af0_18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:15:07,752 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929522f2882d0c446368e5d6d597f24a663_18784ff25b183a4d3733801ace5c9d09 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929522f2882d0c446368e5d6d597f24a663_18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:15:07,753 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929526ce95b22d04967959e6636526e127c_18784ff25b183a4d3733801ace5c9d09 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929526ce95b22d04967959e6636526e127c_18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:15:07,754 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309295d130c19df8140b6a3e6a475cd724411_18784ff25b183a4d3733801ace5c9d09 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309295d130c19df8140b6a3e6a475cd724411_18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:15:07,755 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309296a381e06005446c1ad02ea27f7fbdf59_18784ff25b183a4d3733801ace5c9d09 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309296a381e06005446c1ad02ea27f7fbdf59_18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:15:07,756 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309298199029a9c854083ba4e3604beb6fe10_18784ff25b183a4d3733801ace5c9d09 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309298199029a9c854083ba4e3604beb6fe10_18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:15:07,757 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023092998e6930a24df491d8fa5623d6351e7e8_18784ff25b183a4d3733801ace5c9d09 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023092998e6930a24df491d8fa5623d6351e7e8_18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:15:07,758 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309299bf7b19785394e269eff3343c4cd7b7a_18784ff25b183a4d3733801ace5c9d09 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309299bf7b19785394e269eff3343c4cd7b7a_18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:15:07,759 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309299d049e1370bc461b82b5e9fc71ff7f97_18784ff25b183a4d3733801ace5c9d09 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309299d049e1370bc461b82b5e9fc71ff7f97_18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:15:07,760 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929a9c3b6c9c5cc4ea6bd698486e6836180_18784ff25b183a4d3733801ace5c9d09 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929a9c3b6c9c5cc4ea6bd698486e6836180_18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:15:07,761 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929b33547c3dac34ae8851cf1220ada2d4f_18784ff25b183a4d3733801ace5c9d09 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929b33547c3dac34ae8851cf1220ada2d4f_18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:15:07,761 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929de594f84e46b44a196dea08fa0522524_18784ff25b183a4d3733801ace5c9d09 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929de594f84e46b44a196dea08fa0522524_18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:15:07,762 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929ef0a407fb5b54c35b2664ef33a158dc7_18784ff25b183a4d3733801ace5c9d09 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929ef0a407fb5b54c35b2664ef33a158dc7_18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:15:07,763 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929f20836bc9dcf43fb8675155f7a5867c2_18784ff25b183a4d3733801ace5c9d09 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929f20836bc9dcf43fb8675155f7a5867c2_18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:15:07,764 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929f5bd456db2214fb999fbdd539304f45c_18784ff25b183a4d3733801ace5c9d09 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929f5bd456db2214fb999fbdd539304f45c_18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:15:07,765 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929f7bef947d2eb4328ac8a8da82f9b1ec8_18784ff25b183a4d3733801ace5c9d09 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929f7bef947d2eb4328ac8a8da82f9b1ec8_18784ff25b183a4d3733801ace5c9d09 2023-09-29T06:15:07,766 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2023-09-29T06:15:07,767 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=128, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2023-09-29T06:15:07,769 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2023-09-29T06:15:07,770 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(387): Removing 'TestAcidGuarantees' descriptor. 2023-09-29T06:15:07,771 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=128, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2023-09-29T06:15:07,771 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(377): Removing 'TestAcidGuarantees' from region states. 2023-09-29T06:15:07,771 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1695968107771"}]},"ts":"9223372036854775807"} 2023-09-29T06:15:07,773 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-09-29T06:15:07,773 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 18784ff25b183a4d3733801ace5c9d09, NAME => 'TestAcidGuarantees,,1695968079801.18784ff25b183a4d3733801ace5c9d09.', STARTKEY => '', ENDKEY => ''}] 2023-09-29T06:15:07,773 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(381): Marking 'TestAcidGuarantees' as deleted. 2023-09-29T06:15:07,773 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1695968107773"}]},"ts":"9223372036854775807"} 2023-09-29T06:15:07,774 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1678): Deleted table TestAcidGuarantees state from META 2023-09-29T06:15:07,776 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(127): Finished pid=128, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2023-09-29T06:15:07,777 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=128, state=SUCCESS; DeleteTableProcedure table=TestAcidGuarantees in 54 msec 2023-09-29T06:15:07,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=128 2023-09-29T06:15:07,825 INFO [Listener at localhost/42843 {}] client.HBaseAdmin$TableFuture(3677): Operation: DELETE, Table Name: default:TestAcidGuarantees, procId: 128 completed 2023-09-29T06:15:07,837 INFO [Listener at localhost/42843 {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithAdaptivePolicy#testMobScanAtomicity Thread=225 (was 218) - Thread LEAK? -, OpenFileDescriptor=574 (was 541) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=608 (was 777), ProcessCount=167 (was 171), AvailableMemoryMB=1418 (was 1532) 2023-09-29T06:15:07,848 INFO [Listener at localhost/42843 {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithAdaptivePolicy#testScanAtomicity Thread=225, OpenFileDescriptor=574, MaxFileDescriptor=60000, SystemLoadAverage=608, ProcessCount=167, AvailableMemoryMB=1418 2023-09-29T06:15:07,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] util.TableDescriptorChecker(342): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2023-09-29T06:15:07,849 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'ADAPTIVE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-29T06:15:07,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=129, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestAcidGuarantees 2023-09-29T06:15:07,851 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=129, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2023-09-29T06:15:07,851 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:15:07,851 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 129 2023-09-29T06:15:07,852 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=129, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-29T06:15:07,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=129 2023-09-29T06:15:07,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=129 2023-09-29T06:15:08,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=129 2023-09-29T06:15:08,258 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'ADAPTIVE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a 2023-09-29T06:15:08,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=129 2023-09-29T06:15:08,663 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(888): Instantiated TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:15:08,663 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1668): Closing 991924a9bdcd984903381a94960f3885, disabling compactions & flushes 2023-09-29T06:15:08,663 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1690): Closing region TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:08,663 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:08,663 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. after waiting 0 ms 2023-09-29T06:15:08,663 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:08,663 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1909): Closed TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:08,663 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1622): Region close journal for 991924a9bdcd984903381a94960f3885: 2023-09-29T06:15:08,664 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=129, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2023-09-29T06:15:08,665 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1695968108664"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1695968108664"}]},"ts":"1695968108664"} 2023-09-29T06:15:08,666 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-29T06:15:08,666 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=129, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-29T06:15:08,666 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968108666"}]},"ts":"1695968108666"} 2023-09-29T06:15:08,667 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2023-09-29T06:15:08,682 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=991924a9bdcd984903381a94960f3885, ASSIGN}] 2023-09-29T06:15:08,683 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=130, ppid=129, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=991924a9bdcd984903381a94960f3885, ASSIGN 2023-09-29T06:15:08,684 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=130, ppid=129, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=991924a9bdcd984903381a94960f3885, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,39783,1695967978138; forceNewPlan=false, retain=false 2023-09-29T06:15:08,835 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=130 updating hbase:meta row=991924a9bdcd984903381a94960f3885, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:08,836 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=131, ppid=130, state=RUNNABLE; OpenRegionProcedure 991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138}] 2023-09-29T06:15:08,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=129 2023-09-29T06:15:08,990 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] handler.AssignRegionHandler(130): Open TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:08,990 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(7254): Opening region: {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} 2023-09-29T06:15:08,991 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 991924a9bdcd984903381a94960f3885 2023-09-29T06:15:08,991 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(888): Instantiated TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:15:08,991 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(7296): checking encryption for 991924a9bdcd984903381a94960f3885 2023-09-29T06:15:08,991 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(7299): checking classloading for 991924a9bdcd984903381a94960f3885 2023-09-29T06:15:08,992 INFO [StoreOpener-991924a9bdcd984903381a94960f3885-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 991924a9bdcd984903381a94960f3885 2023-09-29T06:15:08,993 INFO [StoreOpener-991924a9bdcd984903381a94960f3885-1 {}] regionserver.CompactingMemStore(121): Store=A, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2023-09-29T06:15:08,993 INFO [StoreOpener-991924a9bdcd984903381a94960f3885-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 991924a9bdcd984903381a94960f3885 columnFamilyName A 2023-09-29T06:15:08,993 DEBUG [StoreOpener-991924a9bdcd984903381a94960f3885-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:15:08,993 INFO [StoreOpener-991924a9bdcd984903381a94960f3885-1 {}] regionserver.HStore(324): Store=991924a9bdcd984903381a94960f3885/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:15:08,994 INFO [StoreOpener-991924a9bdcd984903381a94960f3885-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 991924a9bdcd984903381a94960f3885 2023-09-29T06:15:08,994 INFO [StoreOpener-991924a9bdcd984903381a94960f3885-1 {}] regionserver.CompactingMemStore(121): Store=B, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2023-09-29T06:15:08,995 INFO [StoreOpener-991924a9bdcd984903381a94960f3885-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 991924a9bdcd984903381a94960f3885 columnFamilyName B 2023-09-29T06:15:08,995 DEBUG [StoreOpener-991924a9bdcd984903381a94960f3885-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:15:08,995 INFO [StoreOpener-991924a9bdcd984903381a94960f3885-1 {}] regionserver.HStore(324): Store=991924a9bdcd984903381a94960f3885/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:15:08,995 INFO [StoreOpener-991924a9bdcd984903381a94960f3885-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 991924a9bdcd984903381a94960f3885 2023-09-29T06:15:08,996 INFO [StoreOpener-991924a9bdcd984903381a94960f3885-1 {}] regionserver.CompactingMemStore(121): Store=C, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2023-09-29T06:15:08,996 INFO [StoreOpener-991924a9bdcd984903381a94960f3885-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 991924a9bdcd984903381a94960f3885 columnFamilyName C 2023-09-29T06:15:08,996 DEBUG [StoreOpener-991924a9bdcd984903381a94960f3885-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:15:08,996 INFO [StoreOpener-991924a9bdcd984903381a94960f3885-1 {}] regionserver.HStore(324): Store=991924a9bdcd984903381a94960f3885/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:15:08,997 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(1169): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:08,997 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885 2023-09-29T06:15:08,997 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885 2023-09-29T06:15:08,999 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2023-09-29T06:15:09,000 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(1076): writing seq id for 991924a9bdcd984903381a94960f3885 2023-09-29T06:15:09,001 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-29T06:15:09,001 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(1093): Opened 991924a9bdcd984903381a94960f3885; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72976076, jitterRate=0.0874282717704773}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2023-09-29T06:15:09,002 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(992): Region open journal for 991924a9bdcd984903381a94960f3885: 2023-09-29T06:15:09,003 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegionServer(2556): Post open deploy tasks for TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885., pid=131, masterSystemTime=1695968108987 2023-09-29T06:15:09,004 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegionServer(2583): Finished post open deploy task for TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:09,004 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] handler.AssignRegionHandler(158): Opened TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:09,004 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=130 updating hbase:meta row=991924a9bdcd984903381a94960f3885, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:09,006 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=131, resume processing ppid=130 2023-09-29T06:15:09,006 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=131, ppid=130, state=SUCCESS; OpenRegionProcedure 991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 in 169 msec 2023-09-29T06:15:09,007 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=130, resume processing ppid=129 2023-09-29T06:15:09,007 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=130, ppid=129, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=991924a9bdcd984903381a94960f3885, ASSIGN in 324 msec 2023-09-29T06:15:09,007 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=129, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-29T06:15:09,008 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968109007"}]},"ts":"1695968109007"} 2023-09-29T06:15:09,008 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2023-09-29T06:15:09,010 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=129, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2023-09-29T06:15:09,011 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=129, state=SUCCESS; CreateTableProcedure table=TestAcidGuarantees in 1.1600 sec 2023-09-29T06:15:09,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=129 2023-09-29T06:15:09,956 INFO [Listener at localhost/42843 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: default:TestAcidGuarantees, procId: 129 completed 2023-09-29T06:15:09,957 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x7e27b061 to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:15:09,962 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b0330f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:15:09,963 DEBUG [Listener at localhost/42843 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:15:09,964 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:42376, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:15:09,965 DEBUG [Listener at localhost/42843 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-29T06:15:09,967 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:58894, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-29T06:15:09,968 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x3a81d834 to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:15:09,972 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79e6d3bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:15:09,973 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x67229aff to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:15:09,980 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53d6be57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:15:09,980 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x02f8d317 to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:15:09,984 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71fe54d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:15:09,985 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x1c70815b to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:15:09,988 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@585499a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:15:09,989 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x2f0274e0 to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:15:09,992 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37605202, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:15:09,992 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x558504ad to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:15:09,997 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6040bc5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:15:09,998 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0c11a915 to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:15:10,002 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71cc7456, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:15:10,002 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x04b2228d to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:15:10,006 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ea6a0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:15:10,006 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x5f369608 to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:15:10,012 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a856129, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:15:10,013 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x3ddf9409 to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:15:10,017 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5632d36c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:15:10,026 DEBUG [hconnection-0x7a30ba71-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:15:10,026 DEBUG [hconnection-0x20f1f083-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:15:10,026 DEBUG [hconnection-0x7463a304-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:15:10,026 DEBUG [hconnection-0x2feb6335-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:15:10,026 DEBUG [hconnection-0x5c425787-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:15:10,027 DEBUG [hconnection-0x68c1c59f-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:15:10,028 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:15:10,028 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:42388, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:15:10,028 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:42414, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:15:10,030 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:42386, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:15:10,030 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:42404, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:15:10,030 DEBUG [hconnection-0x6db617af-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:15:10,030 DEBUG [hconnection-0x55dc1de1-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:15:10,030 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:42426, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:15:10,035 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:42434, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:15:10,035 DEBUG [hconnection-0x2a41ab4b-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:15:10,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=132, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=132, table=TestAcidGuarantees 2023-09-29T06:15:10,036 DEBUG [hconnection-0x390e1e7f-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:15:10,037 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:42450, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:15:10,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=132 2023-09-29T06:15:10,038 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:42462, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:15:10,038 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:42480, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:15:10,044 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=132, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=132, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:15:10,044 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:42472, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:15:10,045 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=132, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=132, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:15:10,045 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=133, ppid=132, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:15:10,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 991924a9bdcd984903381a94960f3885 2023-09-29T06:15:10,058 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 991924a9bdcd984903381a94960f3885 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2023-09-29T06:15:10,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=A 2023-09-29T06:15:10,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:10,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=B 2023-09-29T06:15:10,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:10,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=C 2023-09-29T06:15:10,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:10,077 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/90b9c7b982ef4e069a3a190f97e18d5f is 50, key is test_row_0/A:col10/1695968110058/Put/seqid=0 2023-09-29T06:15:10,087 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/90b9c7b982ef4e069a3a190f97e18d5f 2023-09-29T06:15:10,101 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/c0fc6f4c340242b784b3400ef7ec3fb4 is 50, key is test_row_0/B:col10/1695968110058/Put/seqid=0 2023-09-29T06:15:10,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:10,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968170077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:10,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:10,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968170078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:10,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:10,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968170109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:10,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:10,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968170109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:10,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:10,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968170109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:10,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=132 2023-09-29T06:15:10,207 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=133 2023-09-29T06:15:10,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:10,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. as already flushing 2023-09-29T06:15:10,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:10,208 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] handler.RSProcedureHandler(53): pid=133 java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:10,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=133 java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:10,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=133 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:10,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:10,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968170210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:10,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:10,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968170210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:10,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:10,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968170213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:10,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:10,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968170214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:10,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:10,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968170214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:10,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=132 2023-09-29T06:15:10,360 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=133 2023-09-29T06:15:10,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:10,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. as already flushing 2023-09-29T06:15:10,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:10,360 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] handler.RSProcedureHandler(53): pid=133 java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:10,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=133 java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:10,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=133 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:10,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:10,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968170414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:10,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:10,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968170415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:10,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:10,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968170417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:10,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:10,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968170417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:10,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:10,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968170418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:10,505 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/c0fc6f4c340242b784b3400ef7ec3fb4 2023-09-29T06:15:10,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=133 2023-09-29T06:15:10,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:10,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. as already flushing 2023-09-29T06:15:10,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:10,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] handler.RSProcedureHandler(53): pid=133 java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:10,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=133 java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:10,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=133 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:10,522 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/2d3917acb9b941beb38bfc679a440e26 is 50, key is test_row_0/C:col10/1695968110058/Put/seqid=0 2023-09-29T06:15:10,526 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/2d3917acb9b941beb38bfc679a440e26 2023-09-29T06:15:10,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/90b9c7b982ef4e069a3a190f97e18d5f as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/90b9c7b982ef4e069a3a190f97e18d5f 2023-09-29T06:15:10,533 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/90b9c7b982ef4e069a3a190f97e18d5f, entries=150, sequenceid=15, filesize=11.7 K 2023-09-29T06:15:10,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/c0fc6f4c340242b784b3400ef7ec3fb4 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/c0fc6f4c340242b784b3400ef7ec3fb4 2023-09-29T06:15:10,537 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/c0fc6f4c340242b784b3400ef7ec3fb4, entries=150, sequenceid=15, filesize=11.7 K 2023-09-29T06:15:10,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/2d3917acb9b941beb38bfc679a440e26 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/2d3917acb9b941beb38bfc679a440e26 2023-09-29T06:15:10,543 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/2d3917acb9b941beb38bfc679a440e26, entries=150, sequenceid=15, filesize=11.7 K 2023-09-29T06:15:10,544 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 991924a9bdcd984903381a94960f3885 in 486ms, sequenceid=15, compaction requested=false 2023-09-29T06:15:10,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 991924a9bdcd984903381a94960f3885: 2023-09-29T06:15:10,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=132 2023-09-29T06:15:10,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=133 2023-09-29T06:15:10,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:10,665 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(2819): Flushing 991924a9bdcd984903381a94960f3885 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2023-09-29T06:15:10,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=A 2023-09-29T06:15:10,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:10,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=B 2023-09-29T06:15:10,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:10,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=C 2023-09-29T06:15:10,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:10,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/4a1adca10e4145b4b7ba9146196abc32 is 50, key is test_row_0/A:col10/1695968110075/Put/seqid=0 2023-09-29T06:15:10,674 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/4a1adca10e4145b4b7ba9146196abc32 2023-09-29T06:15:10,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/4cfbf100b19b40a093f6b24a0f5a2ce5 is 50, key is test_row_0/B:col10/1695968110075/Put/seqid=0 2023-09-29T06:15:10,686 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/4cfbf100b19b40a093f6b24a0f5a2ce5 2023-09-29T06:15:10,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/20a3587796874d6ba5bbdf5ad0e5b3c6 is 50, key is test_row_0/C:col10/1695968110075/Put/seqid=0 2023-09-29T06:15:10,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 991924a9bdcd984903381a94960f3885 2023-09-29T06:15:10,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. as already flushing 2023-09-29T06:15:10,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:10,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968170727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:10,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:10,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968170729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:10,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:10,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968170730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:10,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:10,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968170730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:10,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:10,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968170731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:10,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:10,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968170831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:10,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:10,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968170832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:10,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:10,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968170836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:10,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:10,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968170838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:10,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:10,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968170838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:11,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:11,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968171034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:11,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:11,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968171036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:11,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:11,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968171040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:11,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:11,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968171041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:11,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:11,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968171041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:11,099 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/20a3587796874d6ba5bbdf5ad0e5b3c6 2023-09-29T06:15:11,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/4a1adca10e4145b4b7ba9146196abc32 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/4a1adca10e4145b4b7ba9146196abc32 2023-09-29T06:15:11,107 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/4a1adca10e4145b4b7ba9146196abc32, entries=150, sequenceid=37, filesize=11.7 K 2023-09-29T06:15:11,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/4cfbf100b19b40a093f6b24a0f5a2ce5 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/4cfbf100b19b40a093f6b24a0f5a2ce5 2023-09-29T06:15:11,111 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/4cfbf100b19b40a093f6b24a0f5a2ce5, entries=150, sequenceid=37, filesize=11.7 K 2023-09-29T06:15:11,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/20a3587796874d6ba5bbdf5ad0e5b3c6 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/20a3587796874d6ba5bbdf5ad0e5b3c6 2023-09-29T06:15:11,115 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/20a3587796874d6ba5bbdf5ad0e5b3c6, entries=150, sequenceid=37, filesize=11.7 K 2023-09-29T06:15:11,115 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(3022): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for 991924a9bdcd984903381a94960f3885 in 450ms, sequenceid=37, compaction requested=false 2023-09-29T06:15:11,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(2520): Flush status journal for 991924a9bdcd984903381a94960f3885: 2023-09-29T06:15:11,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:11,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=133 2023-09-29T06:15:11,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=133 2023-09-29T06:15:11,117 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=133, resume processing ppid=132 2023-09-29T06:15:11,117 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=133, ppid=132, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0710 sec 2023-09-29T06:15:11,119 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=132, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=132, table=TestAcidGuarantees in 1.0900 sec 2023-09-29T06:15:11,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=132 2023-09-29T06:15:11,141 INFO [Thread-2713 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 132 completed 2023-09-29T06:15:11,142 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:15:11,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=134, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=134, table=TestAcidGuarantees 2023-09-29T06:15:11,145 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=134, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=134, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:15:11,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=134 2023-09-29T06:15:11,145 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=134, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=134, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:15:11,145 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=135, ppid=134, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:15:11,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=134 2023-09-29T06:15:11,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2023-09-29T06:15:11,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:11,298 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2819): Flushing 991924a9bdcd984903381a94960f3885 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2023-09-29T06:15:11,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=A 2023-09-29T06:15:11,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:11,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=B 2023-09-29T06:15:11,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:11,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=C 2023-09-29T06:15:11,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:11,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/dbbf5d562e944afabb3dbe768dd5bc95 is 50, key is test_row_0/A:col10/1695968110729/Put/seqid=0 2023-09-29T06:15:11,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. as already flushing 2023-09-29T06:15:11,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 991924a9bdcd984903381a94960f3885 2023-09-29T06:15:11,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:11,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968171357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:11,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:11,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968171359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:11,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:11,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968171360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:11,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:11,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968171361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:11,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:11,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968171362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:11,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=134 2023-09-29T06:15:11,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:11,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968171463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:11,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:11,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968171466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:11,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:11,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968171466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:11,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:11,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968171467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:11,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:11,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968171470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:11,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:11,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968171667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:11,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:11,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968171669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:11,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:11,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968171669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:11,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:11,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968171671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:11,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:11,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968171673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:11,706 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/dbbf5d562e944afabb3dbe768dd5bc95 2023-09-29T06:15:11,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/369b8ecf18a94122aec03ddd0cc6d85b is 50, key is test_row_0/B:col10/1695968110729/Put/seqid=0 2023-09-29T06:15:11,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=134 2023-09-29T06:15:11,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:11,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968171971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:11,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:11,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968171973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:11,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:11,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968171974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:11,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:11,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968171975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:11,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:11,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968171980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:12,116 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/369b8ecf18a94122aec03ddd0cc6d85b 2023-09-29T06:15:12,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/6f66805fceeb4fe6a8acbc6a78efb37b is 50, key is test_row_0/C:col10/1695968110729/Put/seqid=0 2023-09-29T06:15:12,126 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/6f66805fceeb4fe6a8acbc6a78efb37b 2023-09-29T06:15:12,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/dbbf5d562e944afabb3dbe768dd5bc95 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/dbbf5d562e944afabb3dbe768dd5bc95 2023-09-29T06:15:12,133 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/dbbf5d562e944afabb3dbe768dd5bc95, entries=150, sequenceid=52, filesize=11.7 K 2023-09-29T06:15:12,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/369b8ecf18a94122aec03ddd0cc6d85b as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/369b8ecf18a94122aec03ddd0cc6d85b 2023-09-29T06:15:12,136 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/369b8ecf18a94122aec03ddd0cc6d85b, entries=150, sequenceid=52, filesize=11.7 K 2023-09-29T06:15:12,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/6f66805fceeb4fe6a8acbc6a78efb37b as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/6f66805fceeb4fe6a8acbc6a78efb37b 2023-09-29T06:15:12,140 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/6f66805fceeb4fe6a8acbc6a78efb37b, entries=150, sequenceid=52, filesize=11.7 K 2023-09-29T06:15:12,141 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(3022): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 991924a9bdcd984903381a94960f3885 in 843ms, sequenceid=52, compaction requested=true 2023-09-29T06:15:12,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2520): Flush status journal for 991924a9bdcd984903381a94960f3885: 2023-09-29T06:15:12,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:12,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=135 2023-09-29T06:15:12,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=135 2023-09-29T06:15:12,143 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=135, resume processing ppid=134 2023-09-29T06:15:12,143 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=135, ppid=134, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 997 msec 2023-09-29T06:15:12,145 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=134, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=134, table=TestAcidGuarantees in 1.0010 sec 2023-09-29T06:15:12,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=134 2023-09-29T06:15:12,248 INFO [Thread-2713 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 134 completed 2023-09-29T06:15:12,249 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:15:12,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=136, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=136, table=TestAcidGuarantees 2023-09-29T06:15:12,253 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=136, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=136, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:15:12,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=136 2023-09-29T06:15:12,254 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=136, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=136, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:15:12,256 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:15:12,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=136 2023-09-29T06:15:12,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2023-09-29T06:15:12,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:12,409 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2819): Flushing 991924a9bdcd984903381a94960f3885 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2023-09-29T06:15:12,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=A 2023-09-29T06:15:12,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:12,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=B 2023-09-29T06:15:12,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:12,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=C 2023-09-29T06:15:12,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:12,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/edea0ed8fbdd4721904a93767dbf0d7b is 50, key is test_row_0/A:col10/1695968111359/Put/seqid=0 2023-09-29T06:15:12,417 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/edea0ed8fbdd4721904a93767dbf0d7b 2023-09-29T06:15:12,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/32b0a6415d7e4e6f8d6cdc3360236926 is 50, key is test_row_0/B:col10/1695968111359/Put/seqid=0 2023-09-29T06:15:12,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 991924a9bdcd984903381a94960f3885 2023-09-29T06:15:12,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. as already flushing 2023-09-29T06:15:12,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:12,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968172513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:12,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:12,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968172513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:12,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:12,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968172518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:12,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:12,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968172527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:12,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:12,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968172528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:12,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=136 2023-09-29T06:15:12,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:12,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968172629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:12,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:12,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968172629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:12,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:12,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968172630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:12,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:12,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968172633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:12,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:12,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968172633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:12,826 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/32b0a6415d7e4e6f8d6cdc3360236926 2023-09-29T06:15:12,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/23ac67f70863430784c7071046242f7e is 50, key is test_row_0/C:col10/1695968111359/Put/seqid=0 2023-09-29T06:15:12,835 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/23ac67f70863430784c7071046242f7e 2023-09-29T06:15:12,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:12,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968172834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:12,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/edea0ed8fbdd4721904a93767dbf0d7b as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/edea0ed8fbdd4721904a93767dbf0d7b 2023-09-29T06:15:12,842 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/edea0ed8fbdd4721904a93767dbf0d7b, entries=150, sequenceid=73, filesize=11.7 K 2023-09-29T06:15:12,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/32b0a6415d7e4e6f8d6cdc3360236926 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/32b0a6415d7e4e6f8d6cdc3360236926 2023-09-29T06:15:12,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:12,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968172838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:12,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:12,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968172839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:12,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:12,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968172839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:12,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:12,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968172840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:12,847 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/32b0a6415d7e4e6f8d6cdc3360236926, entries=150, sequenceid=73, filesize=11.7 K 2023-09-29T06:15:12,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/23ac67f70863430784c7071046242f7e as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/23ac67f70863430784c7071046242f7e 2023-09-29T06:15:12,850 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/23ac67f70863430784c7071046242f7e, entries=150, sequenceid=73, filesize=11.7 K 2023-09-29T06:15:12,851 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(3022): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 991924a9bdcd984903381a94960f3885 in 442ms, sequenceid=73, compaction requested=true 2023-09-29T06:15:12,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2520): Flush status journal for 991924a9bdcd984903381a94960f3885: 2023-09-29T06:15:12,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:12,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=137 2023-09-29T06:15:12,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=137 2023-09-29T06:15:12,855 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=137, resume processing ppid=136 2023-09-29T06:15:12,855 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=137, ppid=136, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 598 msec 2023-09-29T06:15:12,856 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=136, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=136, table=TestAcidGuarantees in 606 msec 2023-09-29T06:15:12,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=136 2023-09-29T06:15:12,856 INFO [Thread-2713 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 136 completed 2023-09-29T06:15:12,857 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:15:12,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=138, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees 2023-09-29T06:15:12,859 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=138, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:15:12,860 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=138, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:15:12,860 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:15:12,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=138 2023-09-29T06:15:12,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=138 2023-09-29T06:15:12,964 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2023-09-29T06:15:13,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2023-09-29T06:15:13,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:13,013 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2819): Flushing 991924a9bdcd984903381a94960f3885 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2023-09-29T06:15:13,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=A 2023-09-29T06:15:13,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:13,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=B 2023-09-29T06:15:13,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:13,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=C 2023-09-29T06:15:13,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:13,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/168da897cfd04ef0964d1d5f699e1d86 is 50, key is test_row_0/A:col10/1695968112517/Put/seqid=0 2023-09-29T06:15:13,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 991924a9bdcd984903381a94960f3885 2023-09-29T06:15:13,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. as already flushing 2023-09-29T06:15:13,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=138 2023-09-29T06:15:13,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:13,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968173167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:13,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:13,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968173168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:13,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:13,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968173169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:13,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:13,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968173169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:13,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:13,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968173170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:13,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:13,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968173276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:13,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:13,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968173277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:13,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:13,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968173277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:13,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:13,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968173277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:13,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:13,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968173277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:13,420 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/168da897cfd04ef0964d1d5f699e1d86 2023-09-29T06:15:13,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/639201b8a8744301bb0506f2ca309136 is 50, key is test_row_0/B:col10/1695968112517/Put/seqid=0 2023-09-29T06:15:13,431 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/639201b8a8744301bb0506f2ca309136 2023-09-29T06:15:13,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/6248c31548bd440a9ccd9ffd97d41cb8 is 50, key is test_row_0/C:col10/1695968112517/Put/seqid=0 2023-09-29T06:15:13,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=138 2023-09-29T06:15:13,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:13,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968173481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:13,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:13,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968173481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:13,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:13,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968173481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:13,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:13,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968173482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:13,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:13,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968173482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:13,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:13,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968173785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:13,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:13,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968173786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:13,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:13,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968173787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:13,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:13,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968173788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:13,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:13,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968173788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:13,841 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/6248c31548bd440a9ccd9ffd97d41cb8 2023-09-29T06:15:13,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/168da897cfd04ef0964d1d5f699e1d86 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/168da897cfd04ef0964d1d5f699e1d86 2023-09-29T06:15:13,848 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/168da897cfd04ef0964d1d5f699e1d86, entries=150, sequenceid=88, filesize=11.7 K 2023-09-29T06:15:13,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/639201b8a8744301bb0506f2ca309136 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/639201b8a8744301bb0506f2ca309136 2023-09-29T06:15:13,851 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/639201b8a8744301bb0506f2ca309136, entries=150, sequenceid=88, filesize=11.7 K 2023-09-29T06:15:13,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/6248c31548bd440a9ccd9ffd97d41cb8 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/6248c31548bd440a9ccd9ffd97d41cb8 2023-09-29T06:15:13,855 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/6248c31548bd440a9ccd9ffd97d41cb8, entries=150, sequenceid=88, filesize=11.7 K 2023-09-29T06:15:13,855 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(3022): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 991924a9bdcd984903381a94960f3885 in 843ms, sequenceid=88, compaction requested=true 2023-09-29T06:15:13,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2520): Flush status journal for 991924a9bdcd984903381a94960f3885: 2023-09-29T06:15:13,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:13,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=139 2023-09-29T06:15:13,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=139 2023-09-29T06:15:13,857 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=139, resume processing ppid=138 2023-09-29T06:15:13,857 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=139, ppid=138, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 996 msec 2023-09-29T06:15:13,858 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=138, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees in 1.0010 sec 2023-09-29T06:15:13,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=138 2023-09-29T06:15:13,963 INFO [Thread-2713 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 138 completed 2023-09-29T06:15:13,964 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:15:13,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=140, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees 2023-09-29T06:15:13,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=140 2023-09-29T06:15:13,967 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=140, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:15:13,967 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=140, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:15:13,968 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:15:14,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=140 2023-09-29T06:15:14,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2023-09-29T06:15:14,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:14,121 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2819): Flushing 991924a9bdcd984903381a94960f3885 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2023-09-29T06:15:14,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=A 2023-09-29T06:15:14,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:14,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=B 2023-09-29T06:15:14,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:14,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=C 2023-09-29T06:15:14,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:14,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/d5f6bcf725074747a723de92351c052c is 50, key is test_row_0/A:col10/1695968113169/Put/seqid=0 2023-09-29T06:15:14,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=140 2023-09-29T06:15:14,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 991924a9bdcd984903381a94960f3885 2023-09-29T06:15:14,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. as already flushing 2023-09-29T06:15:14,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:14,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968174307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:14,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:14,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968174308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:14,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:14,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968174308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:14,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:14,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968174309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:14,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:14,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968174312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:14,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:14,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968174414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:14,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:14,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968174414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:14,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:14,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968174416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:14,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:14,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968174416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:14,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:14,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968174416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:14,529 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/d5f6bcf725074747a723de92351c052c 2023-09-29T06:15:14,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/319867d86ceb4dd0911ed15eeeb322fd is 50, key is test_row_0/B:col10/1695968113169/Put/seqid=0 2023-09-29T06:15:14,541 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/319867d86ceb4dd0911ed15eeeb322fd 2023-09-29T06:15:14,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/d58f2e727b304843997bb68ba3a9dcb1 is 50, key is test_row_0/C:col10/1695968113169/Put/seqid=0 2023-09-29T06:15:14,553 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/d58f2e727b304843997bb68ba3a9dcb1 2023-09-29T06:15:14,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/d5f6bcf725074747a723de92351c052c as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/d5f6bcf725074747a723de92351c052c 2023-09-29T06:15:14,560 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/d5f6bcf725074747a723de92351c052c, entries=150, sequenceid=109, filesize=11.7 K 2023-09-29T06:15:14,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/319867d86ceb4dd0911ed15eeeb322fd as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/319867d86ceb4dd0911ed15eeeb322fd 2023-09-29T06:15:14,564 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/319867d86ceb4dd0911ed15eeeb322fd, entries=150, sequenceid=109, filesize=11.7 K 2023-09-29T06:15:14,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/d58f2e727b304843997bb68ba3a9dcb1 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/d58f2e727b304843997bb68ba3a9dcb1 2023-09-29T06:15:14,568 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/d58f2e727b304843997bb68ba3a9dcb1, entries=150, sequenceid=109, filesize=11.7 K 2023-09-29T06:15:14,568 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(3022): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 991924a9bdcd984903381a94960f3885 in 448ms, sequenceid=109, compaction requested=true 2023-09-29T06:15:14,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2520): Flush status journal for 991924a9bdcd984903381a94960f3885: 2023-09-29T06:15:14,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:14,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=141 2023-09-29T06:15:14,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=141 2023-09-29T06:15:14,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=140 2023-09-29T06:15:14,571 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=141, resume processing ppid=140 2023-09-29T06:15:14,571 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=141, ppid=140, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 602 msec 2023-09-29T06:15:14,572 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=140, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees in 607 msec 2023-09-29T06:15:14,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 991924a9bdcd984903381a94960f3885 2023-09-29T06:15:14,624 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 991924a9bdcd984903381a94960f3885 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2023-09-29T06:15:14,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=A 2023-09-29T06:15:14,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:14,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=B 2023-09-29T06:15:14,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:14,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=C 2023-09-29T06:15:14,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:14,628 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/20e9baa79e8149799be4bfdbfb28c5a7 is 50, key is test_row_0/A:col10/1695968114624/Put/seqid=0 2023-09-29T06:15:14,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:14,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968174642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:14,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:14,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968174643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:14,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:14,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968174648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:14,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:14,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968174649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:14,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:14,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968174650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:14,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:14,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968174751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:14,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:14,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968174751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:14,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:14,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968174755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:14,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:14,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968174755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:14,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:14,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968174757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:14,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:14,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968174958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:14,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:14,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968174958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:14,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:14,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968174961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:14,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:14,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968174961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:14,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:14,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968174965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:15,039 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/20e9baa79e8149799be4bfdbfb28c5a7 2023-09-29T06:15:15,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/f672939870fe479eb53209f479039aba is 50, key is test_row_0/B:col10/1695968114624/Put/seqid=0 2023-09-29T06:15:15,048 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/f672939870fe479eb53209f479039aba 2023-09-29T06:15:15,053 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/8d556a8733644627a5c8729e86eff521 is 50, key is test_row_0/C:col10/1695968114624/Put/seqid=0 2023-09-29T06:15:15,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=140 2023-09-29T06:15:15,071 INFO [Thread-2713 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 140 completed 2023-09-29T06:15:15,072 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:15:15,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees 2023-09-29T06:15:15,074 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:15:15,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=142 2023-09-29T06:15:15,074 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=142, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:15:15,075 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:15:15,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=142 2023-09-29T06:15:15,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2023-09-29T06:15:15,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:15,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. as already flushing 2023-09-29T06:15:15,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:15,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(53): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:15,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:15,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:15,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:15,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968175265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:15,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:15,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968175265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:15,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:15,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968175266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:15,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:15,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968175267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:15,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:15,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968175273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:15,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=142 2023-09-29T06:15:15,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2023-09-29T06:15:15,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:15,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. as already flushing 2023-09-29T06:15:15,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:15,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(53): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:15,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:15,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:15,457 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/8d556a8733644627a5c8729e86eff521 2023-09-29T06:15:15,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/20e9baa79e8149799be4bfdbfb28c5a7 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/20e9baa79e8149799be4bfdbfb28c5a7 2023-09-29T06:15:15,464 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/20e9baa79e8149799be4bfdbfb28c5a7, entries=250, sequenceid=126, filesize=16.3 K 2023-09-29T06:15:15,464 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(982): StoreScanner already closing. There is no need to updateReaders 2023-09-29T06:15:15,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/f672939870fe479eb53209f479039aba as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/f672939870fe479eb53209f479039aba 2023-09-29T06:15:15,468 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/f672939870fe479eb53209f479039aba, entries=150, sequenceid=126, filesize=11.7 K 2023-09-29T06:15:15,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/8d556a8733644627a5c8729e86eff521 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/8d556a8733644627a5c8729e86eff521 2023-09-29T06:15:15,472 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/8d556a8733644627a5c8729e86eff521, entries=150, sequenceid=126, filesize=11.7 K 2023-09-29T06:15:15,473 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 991924a9bdcd984903381a94960f3885 in 849ms, sequenceid=126, compaction requested=true 2023-09-29T06:15:15,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 991924a9bdcd984903381a94960f3885: 2023-09-29T06:15:15,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 991924a9bdcd984903381a94960f3885:A, priority=-2147483648, current under compaction store size is 1 2023-09-29T06:15:15,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:15,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 991924a9bdcd984903381a94960f3885:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:15:15,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:15,473 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2023-09-29T06:15:15,473 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2023-09-29T06:15:15,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 991924a9bdcd984903381a94960f3885:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:15:15,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:15:15,476 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 83888 starting at candidate #0 after considering 15 permutations with 15 in ratio 2023-09-29T06:15:15,476 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 991924a9bdcd984903381a94960f3885/B is initiating minor compaction (all files) 2023-09-29T06:15:15,476 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 88568 starting at candidate #0 after considering 15 permutations with 15 in ratio 2023-09-29T06:15:15,477 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 991924a9bdcd984903381a94960f3885/B in TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:15,477 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 991924a9bdcd984903381a94960f3885/A is initiating minor compaction (all files) 2023-09-29T06:15:15,477 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/c0fc6f4c340242b784b3400ef7ec3fb4, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/4cfbf100b19b40a093f6b24a0f5a2ce5, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/369b8ecf18a94122aec03ddd0cc6d85b, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/32b0a6415d7e4e6f8d6cdc3360236926, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/639201b8a8744301bb0506f2ca309136, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/319867d86ceb4dd0911ed15eeeb322fd, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/f672939870fe479eb53209f479039aba] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp, totalSize=81.9 K 2023-09-29T06:15:15,477 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 991924a9bdcd984903381a94960f3885/A in TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:15,477 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/90b9c7b982ef4e069a3a190f97e18d5f, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/4a1adca10e4145b4b7ba9146196abc32, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/dbbf5d562e944afabb3dbe768dd5bc95, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/edea0ed8fbdd4721904a93767dbf0d7b, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/168da897cfd04ef0964d1d5f699e1d86, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/d5f6bcf725074747a723de92351c052c, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/20e9baa79e8149799be4bfdbfb28c5a7] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp, totalSize=86.5 K 2023-09-29T06:15:15,477 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting c0fc6f4c340242b784b3400ef7ec3fb4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1695968110051 2023-09-29T06:15:15,477 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 90b9c7b982ef4e069a3a190f97e18d5f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1695968110051 2023-09-29T06:15:15,477 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 4cfbf100b19b40a093f6b24a0f5a2ce5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1695968110075 2023-09-29T06:15:15,477 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 4a1adca10e4145b4b7ba9146196abc32, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1695968110075 2023-09-29T06:15:15,478 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 369b8ecf18a94122aec03ddd0cc6d85b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1695968110723 2023-09-29T06:15:15,478 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting dbbf5d562e944afabb3dbe768dd5bc95, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1695968110723 2023-09-29T06:15:15,478 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 32b0a6415d7e4e6f8d6cdc3360236926, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1695968111354 2023-09-29T06:15:15,478 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting edea0ed8fbdd4721904a93767dbf0d7b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1695968111354 2023-09-29T06:15:15,478 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 639201b8a8744301bb0506f2ca309136, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1695968112487 2023-09-29T06:15:15,478 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 168da897cfd04ef0964d1d5f699e1d86, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1695968112487 2023-09-29T06:15:15,478 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 319867d86ceb4dd0911ed15eeeb322fd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1695968113167 2023-09-29T06:15:15,479 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting d5f6bcf725074747a723de92351c052c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1695968113167 2023-09-29T06:15:15,479 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting f672939870fe479eb53209f479039aba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1695968114308 2023-09-29T06:15:15,479 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 20e9baa79e8149799be4bfdbfb28c5a7, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1695968114299 2023-09-29T06:15:15,492 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 991924a9bdcd984903381a94960f3885#B#compaction#454 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:15,492 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 991924a9bdcd984903381a94960f3885#A#compaction#453 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:15,492 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/96c529cd4b1a4af6b40900609c1bc76f is 50, key is test_row_0/B:col10/1695968114624/Put/seqid=0 2023-09-29T06:15:15,492 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/64a833b80135463f9446760ac90da910 is 50, key is test_row_0/A:col10/1695968114624/Put/seqid=0 2023-09-29T06:15:15,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2023-09-29T06:15:15,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:15,533 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2819): Flushing 991924a9bdcd984903381a94960f3885 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2023-09-29T06:15:15,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=A 2023-09-29T06:15:15,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:15,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=B 2023-09-29T06:15:15,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:15,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=C 2023-09-29T06:15:15,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:15,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/1007f1156b124f05a80a79e0ac53edd8 is 50, key is test_row_0/A:col10/1695968114639/Put/seqid=0 2023-09-29T06:15:15,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=142 2023-09-29T06:15:15,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. as already flushing 2023-09-29T06:15:15,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 991924a9bdcd984903381a94960f3885 2023-09-29T06:15:15,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:15,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968175812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:15,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:15,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968175813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:15,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:15,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968175813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:15,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:15,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968175821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:15,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:15,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968175822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:15,904 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/64a833b80135463f9446760ac90da910 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/64a833b80135463f9446760ac90da910 2023-09-29T06:15:15,904 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/96c529cd4b1a4af6b40900609c1bc76f as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/96c529cd4b1a4af6b40900609c1bc76f 2023-09-29T06:15:15,909 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 7 (all) file(s) in 991924a9bdcd984903381a94960f3885/A of 991924a9bdcd984903381a94960f3885 into 64a833b80135463f9446760ac90da910(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:15,909 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 991924a9bdcd984903381a94960f3885: 2023-09-29T06:15:15,909 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885., storeName=991924a9bdcd984903381a94960f3885/A, priority=9, startTime=1695968115473; duration=0sec 2023-09-29T06:15:15,909 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:15:15,909 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 991924a9bdcd984903381a94960f3885:A 2023-09-29T06:15:15,909 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2023-09-29T06:15:15,909 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 7 (all) file(s) in 991924a9bdcd984903381a94960f3885/B of 991924a9bdcd984903381a94960f3885 into 96c529cd4b1a4af6b40900609c1bc76f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:15,909 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 991924a9bdcd984903381a94960f3885: 2023-09-29T06:15:15,909 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885., storeName=991924a9bdcd984903381a94960f3885/B, priority=9, startTime=1695968115473; duration=0sec 2023-09-29T06:15:15,910 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:15,910 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 991924a9bdcd984903381a94960f3885:B 2023-09-29T06:15:15,911 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 83888 starting at candidate #0 after considering 15 permutations with 15 in ratio 2023-09-29T06:15:15,911 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 991924a9bdcd984903381a94960f3885/C is initiating minor compaction (all files) 2023-09-29T06:15:15,911 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 991924a9bdcd984903381a94960f3885/C in TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:15,911 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/2d3917acb9b941beb38bfc679a440e26, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/20a3587796874d6ba5bbdf5ad0e5b3c6, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/6f66805fceeb4fe6a8acbc6a78efb37b, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/23ac67f70863430784c7071046242f7e, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/6248c31548bd440a9ccd9ffd97d41cb8, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/d58f2e727b304843997bb68ba3a9dcb1, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/8d556a8733644627a5c8729e86eff521] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp, totalSize=81.9 K 2023-09-29T06:15:15,912 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 2d3917acb9b941beb38bfc679a440e26, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1695968110051 2023-09-29T06:15:15,912 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 20a3587796874d6ba5bbdf5ad0e5b3c6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1695968110075 2023-09-29T06:15:15,912 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 6f66805fceeb4fe6a8acbc6a78efb37b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1695968110723 2023-09-29T06:15:15,913 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 23ac67f70863430784c7071046242f7e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1695968111354 2023-09-29T06:15:15,913 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 6248c31548bd440a9ccd9ffd97d41cb8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1695968112487 2023-09-29T06:15:15,913 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting d58f2e727b304843997bb68ba3a9dcb1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1695968113167 2023-09-29T06:15:15,913 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 8d556a8733644627a5c8729e86eff521, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1695968114308 2023-09-29T06:15:15,925 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 991924a9bdcd984903381a94960f3885#C#compaction#456 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:15,925 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/2c3d35cbce1d4e6a8f683b9a3d6eceb6 is 50, key is test_row_0/C:col10/1695968114624/Put/seqid=0 2023-09-29T06:15:15,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:15,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968175923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:15,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:15,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968175923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:15,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:15,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968175923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:15,932 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/2c3d35cbce1d4e6a8f683b9a3d6eceb6 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/2c3d35cbce1d4e6a8f683b9a3d6eceb6 2023-09-29T06:15:15,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:15,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968175930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:15,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:15,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968175930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:15,937 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 7 (all) file(s) in 991924a9bdcd984903381a94960f3885/C of 991924a9bdcd984903381a94960f3885 into 2c3d35cbce1d4e6a8f683b9a3d6eceb6(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:15,937 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 991924a9bdcd984903381a94960f3885: 2023-09-29T06:15:15,937 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885., storeName=991924a9bdcd984903381a94960f3885/C, priority=9, startTime=1695968115473; duration=0sec 2023-09-29T06:15:15,937 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:15,937 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 991924a9bdcd984903381a94960f3885:C 2023-09-29T06:15:15,940 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/1007f1156b124f05a80a79e0ac53edd8 2023-09-29T06:15:15,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/17097b1a029b4864bf7e62c146e394a7 is 50, key is test_row_0/B:col10/1695968114639/Put/seqid=0 2023-09-29T06:15:16,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:16,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968176129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:16,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:16,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968176129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:16,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:16,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968176130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:16,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:16,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968176135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:16,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:16,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968176136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:16,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=142 2023-09-29T06:15:16,350 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/17097b1a029b4864bf7e62c146e394a7 2023-09-29T06:15:16,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/3cfbb791e1af43da8506e700a119534e is 50, key is test_row_0/C:col10/1695968114639/Put/seqid=0 2023-09-29T06:15:16,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:16,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968176433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:16,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:16,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968176434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:16,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:16,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968176434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:16,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:16,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968176437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:16,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:16,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968176439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:16,760 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/3cfbb791e1af43da8506e700a119534e 2023-09-29T06:15:16,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/1007f1156b124f05a80a79e0ac53edd8 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/1007f1156b124f05a80a79e0ac53edd8 2023-09-29T06:15:16,768 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/1007f1156b124f05a80a79e0ac53edd8, entries=150, sequenceid=145, filesize=11.8 K 2023-09-29T06:15:16,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/17097b1a029b4864bf7e62c146e394a7 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/17097b1a029b4864bf7e62c146e394a7 2023-09-29T06:15:16,772 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/17097b1a029b4864bf7e62c146e394a7, entries=150, sequenceid=145, filesize=11.8 K 2023-09-29T06:15:16,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/3cfbb791e1af43da8506e700a119534e as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/3cfbb791e1af43da8506e700a119534e 2023-09-29T06:15:16,776 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/3cfbb791e1af43da8506e700a119534e, entries=150, sequenceid=145, filesize=11.8 K 2023-09-29T06:15:16,777 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(3022): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 991924a9bdcd984903381a94960f3885 in 1244ms, sequenceid=145, compaction requested=false 2023-09-29T06:15:16,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2520): Flush status journal for 991924a9bdcd984903381a94960f3885: 2023-09-29T06:15:16,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:16,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=143 2023-09-29T06:15:16,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=143 2023-09-29T06:15:16,779 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=143, resume processing ppid=142 2023-09-29T06:15:16,779 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=143, ppid=142, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7030 sec 2023-09-29T06:15:16,780 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=142, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees in 1.7070 sec 2023-09-29T06:15:16,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 991924a9bdcd984903381a94960f3885 2023-09-29T06:15:16,946 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 991924a9bdcd984903381a94960f3885 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2023-09-29T06:15:16,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=A 2023-09-29T06:15:16,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:16,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=B 2023-09-29T06:15:16,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:16,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=C 2023-09-29T06:15:16,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:16,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/9c35a32d1cc147028be1d562a6929f56 is 50, key is test_row_0/A:col10/1695968116945/Put/seqid=0 2023-09-29T06:15:16,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:16,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968176959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:16,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:16,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968176960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:16,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:16,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968176961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:16,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:16,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968176961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:16,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:16,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968176964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:17,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:17,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968177066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:17,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:17,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968177066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:17,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:17,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968177067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:17,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:17,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968177067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:17,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:17,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968177071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:17,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=142 2023-09-29T06:15:17,178 INFO [Thread-2713 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 142 completed 2023-09-29T06:15:17,180 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:15:17,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees 2023-09-29T06:15:17,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=144 2023-09-29T06:15:17,182 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:15:17,183 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=144, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:15:17,183 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:15:17,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:17,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968177269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:17,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:17,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968177271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:17,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:17,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968177271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:17,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:17,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968177272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:17,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:17,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968177276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:17,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=144 2023-09-29T06:15:17,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2023-09-29T06:15:17,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:17,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. as already flushing 2023-09-29T06:15:17,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:17,336 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(53): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:17,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:17,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:17,354 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/9c35a32d1cc147028be1d562a6929f56 2023-09-29T06:15:17,360 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/935edb774977402f88922d018f561624 is 50, key is test_row_0/B:col10/1695968116945/Put/seqid=0 2023-09-29T06:15:17,363 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/935edb774977402f88922d018f561624 2023-09-29T06:15:17,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/bd00c6dc870b418e8bbd9551404a6318 is 50, key is test_row_0/C:col10/1695968116945/Put/seqid=0 2023-09-29T06:15:17,460 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2023-09-29T06:15:17,460 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2023-09-29T06:15:17,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=144 2023-09-29T06:15:17,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2023-09-29T06:15:17,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:17,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. as already flushing 2023-09-29T06:15:17,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:17,488 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(53): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:17,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:17,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:17,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:17,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968177576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:17,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:17,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968177576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:17,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:17,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968177577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:17,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:17,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968177577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:17,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:17,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968177581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:17,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2023-09-29T06:15:17,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:17,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. as already flushing 2023-09-29T06:15:17,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:17,641 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(53): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:17,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:17,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:17,771 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/bd00c6dc870b418e8bbd9551404a6318 2023-09-29T06:15:17,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/9c35a32d1cc147028be1d562a6929f56 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/9c35a32d1cc147028be1d562a6929f56 2023-09-29T06:15:17,779 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/9c35a32d1cc147028be1d562a6929f56, entries=150, sequenceid=166, filesize=11.8 K 2023-09-29T06:15:17,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/935edb774977402f88922d018f561624 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/935edb774977402f88922d018f561624 2023-09-29T06:15:17,783 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/935edb774977402f88922d018f561624, entries=150, sequenceid=166, filesize=11.8 K 2023-09-29T06:15:17,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/bd00c6dc870b418e8bbd9551404a6318 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/bd00c6dc870b418e8bbd9551404a6318 2023-09-29T06:15:17,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=144 2023-09-29T06:15:17,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/bd00c6dc870b418e8bbd9551404a6318, entries=150, sequenceid=166, filesize=11.8 K 2023-09-29T06:15:17,788 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 991924a9bdcd984903381a94960f3885 in 842ms, sequenceid=166, compaction requested=true 2023-09-29T06:15:17,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 991924a9bdcd984903381a94960f3885: 2023-09-29T06:15:17,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 991924a9bdcd984903381a94960f3885:A, priority=-2147483648, current under compaction store size is 1 2023-09-29T06:15:17,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:17,788 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:15:17,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 991924a9bdcd984903381a94960f3885:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:15:17,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:17,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 991924a9bdcd984903381a94960f3885:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:15:17,788 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:15:17,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:15:17,789 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36492 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:15:17,789 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 991924a9bdcd984903381a94960f3885/A is initiating minor compaction (all files) 2023-09-29T06:15:17,789 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36492 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:15:17,789 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 991924a9bdcd984903381a94960f3885/A in TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:17,789 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 991924a9bdcd984903381a94960f3885/B is initiating minor compaction (all files) 2023-09-29T06:15:17,789 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 991924a9bdcd984903381a94960f3885/B in TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:17,789 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/64a833b80135463f9446760ac90da910, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/1007f1156b124f05a80a79e0ac53edd8, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/9c35a32d1cc147028be1d562a6929f56] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp, totalSize=35.6 K 2023-09-29T06:15:17,789 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/96c529cd4b1a4af6b40900609c1bc76f, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/17097b1a029b4864bf7e62c146e394a7, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/935edb774977402f88922d018f561624] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp, totalSize=35.6 K 2023-09-29T06:15:17,789 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 64a833b80135463f9446760ac90da910, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1695968114308 2023-09-29T06:15:17,789 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 96c529cd4b1a4af6b40900609c1bc76f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1695968114308 2023-09-29T06:15:17,790 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 1007f1156b124f05a80a79e0ac53edd8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1695968114639 2023-09-29T06:15:17,790 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 17097b1a029b4864bf7e62c146e394a7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1695968114639 2023-09-29T06:15:17,790 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 9c35a32d1cc147028be1d562a6929f56, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1695968115812 2023-09-29T06:15:17,790 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 935edb774977402f88922d018f561624, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1695968115812 2023-09-29T06:15:17,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2023-09-29T06:15:17,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:17,793 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2819): Flushing 991924a9bdcd984903381a94960f3885 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2023-09-29T06:15:17,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=A 2023-09-29T06:15:17,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:17,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=B 2023-09-29T06:15:17,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:17,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=C 2023-09-29T06:15:17,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:17,797 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 991924a9bdcd984903381a94960f3885#B#compaction#462 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:17,797 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/f3bca8db01df42039df93f968704b5f8 is 50, key is test_row_0/B:col10/1695968116945/Put/seqid=0 2023-09-29T06:15:17,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/c5476dcba8e64f5fa402f88008b64ff3 is 50, key is test_row_0/A:col10/1695968116963/Put/seqid=0 2023-09-29T06:15:17,798 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 991924a9bdcd984903381a94960f3885#A#compaction#464 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:17,798 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/e711c8235f31492eba6f558df55797b5 is 50, key is test_row_0/A:col10/1695968116945/Put/seqid=0 2023-09-29T06:15:18,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. as already flushing 2023-09-29T06:15:18,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 991924a9bdcd984903381a94960f3885 2023-09-29T06:15:18,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:18,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968178098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:18,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:18,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968178098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:18,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:18,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968178101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:18,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:18,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968178102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:18,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:18,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968178103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:18,206 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/f3bca8db01df42039df93f968704b5f8 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/f3bca8db01df42039df93f968704b5f8 2023-09-29T06:15:18,208 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/c5476dcba8e64f5fa402f88008b64ff3 2023-09-29T06:15:18,210 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 991924a9bdcd984903381a94960f3885/B of 991924a9bdcd984903381a94960f3885 into f3bca8db01df42039df93f968704b5f8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:18,211 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 991924a9bdcd984903381a94960f3885: 2023-09-29T06:15:18,211 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885., storeName=991924a9bdcd984903381a94960f3885/B, priority=13, startTime=1695968117788; duration=0sec 2023-09-29T06:15:18,211 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:15:18,211 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 991924a9bdcd984903381a94960f3885:B 2023-09-29T06:15:18,211 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:15:18,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:18,212 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36492 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:15:18,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968178208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:18,212 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 991924a9bdcd984903381a94960f3885/C is initiating minor compaction (all files) 2023-09-29T06:15:18,212 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 991924a9bdcd984903381a94960f3885/C in TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:18,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:18,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968178208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:18,212 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/2c3d35cbce1d4e6a8f683b9a3d6eceb6, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/3cfbb791e1af43da8506e700a119534e, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/bd00c6dc870b418e8bbd9551404a6318] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp, totalSize=35.6 K 2023-09-29T06:15:18,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:18,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968178209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:18,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:18,213 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 2c3d35cbce1d4e6a8f683b9a3d6eceb6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1695968114308 2023-09-29T06:15:18,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968178209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:18,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:18,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968178209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:18,214 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 3cfbb791e1af43da8506e700a119534e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1695968114639 2023-09-29T06:15:18,214 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting bd00c6dc870b418e8bbd9551404a6318, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1695968115812 2023-09-29T06:15:18,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/959ca23e26c8448aafd0380ca8dc4406 is 50, key is test_row_0/B:col10/1695968116963/Put/seqid=0 2023-09-29T06:15:18,217 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/e711c8235f31492eba6f558df55797b5 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/e711c8235f31492eba6f558df55797b5 2023-09-29T06:15:18,222 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 991924a9bdcd984903381a94960f3885#C#compaction#466 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:18,223 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/8d927e6b7d72406ba1a62e38271543f4 is 50, key is test_row_0/C:col10/1695968116945/Put/seqid=0 2023-09-29T06:15:18,223 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 991924a9bdcd984903381a94960f3885/A of 991924a9bdcd984903381a94960f3885 into e711c8235f31492eba6f558df55797b5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:18,223 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 991924a9bdcd984903381a94960f3885: 2023-09-29T06:15:18,223 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885., storeName=991924a9bdcd984903381a94960f3885/A, priority=13, startTime=1695968117788; duration=0sec 2023-09-29T06:15:18,223 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:18,223 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 991924a9bdcd984903381a94960f3885:A 2023-09-29T06:15:18,225 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/959ca23e26c8448aafd0380ca8dc4406 2023-09-29T06:15:18,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/e9ef77f047eb40f098a173c68a7f226f is 50, key is test_row_0/C:col10/1695968116963/Put/seqid=0 2023-09-29T06:15:18,232 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/8d927e6b7d72406ba1a62e38271543f4 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/8d927e6b7d72406ba1a62e38271543f4 2023-09-29T06:15:18,235 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/e9ef77f047eb40f098a173c68a7f226f 2023-09-29T06:15:18,236 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 991924a9bdcd984903381a94960f3885/C of 991924a9bdcd984903381a94960f3885 into 8d927e6b7d72406ba1a62e38271543f4(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:18,236 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 991924a9bdcd984903381a94960f3885: 2023-09-29T06:15:18,236 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885., storeName=991924a9bdcd984903381a94960f3885/C, priority=13, startTime=1695968117788; duration=0sec 2023-09-29T06:15:18,236 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:18,236 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 991924a9bdcd984903381a94960f3885:C 2023-09-29T06:15:18,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/c5476dcba8e64f5fa402f88008b64ff3 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/c5476dcba8e64f5fa402f88008b64ff3 2023-09-29T06:15:18,242 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/c5476dcba8e64f5fa402f88008b64ff3, entries=150, sequenceid=184, filesize=11.8 K 2023-09-29T06:15:18,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/959ca23e26c8448aafd0380ca8dc4406 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/959ca23e26c8448aafd0380ca8dc4406 2023-09-29T06:15:18,246 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/959ca23e26c8448aafd0380ca8dc4406, entries=150, sequenceid=184, filesize=11.8 K 2023-09-29T06:15:18,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/e9ef77f047eb40f098a173c68a7f226f as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/e9ef77f047eb40f098a173c68a7f226f 2023-09-29T06:15:18,250 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/e9ef77f047eb40f098a173c68a7f226f, entries=150, sequenceid=184, filesize=11.8 K 2023-09-29T06:15:18,250 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(3022): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 991924a9bdcd984903381a94960f3885 in 457ms, sequenceid=184, compaction requested=false 2023-09-29T06:15:18,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2520): Flush status journal for 991924a9bdcd984903381a94960f3885: 2023-09-29T06:15:18,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:18,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=145 2023-09-29T06:15:18,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=145 2023-09-29T06:15:18,253 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=145, resume processing ppid=144 2023-09-29T06:15:18,253 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=145, ppid=144, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0680 sec 2023-09-29T06:15:18,254 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=144, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees in 1.0730 sec 2023-09-29T06:15:18,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=144 2023-09-29T06:15:18,285 INFO [Thread-2713 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 144 completed 2023-09-29T06:15:18,287 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:15:18,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2023-09-29T06:15:18,289 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:15:18,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=146 2023-09-29T06:15:18,290 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:15:18,290 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:15:18,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=146 2023-09-29T06:15:18,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 991924a9bdcd984903381a94960f3885 2023-09-29T06:15:18,418 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 991924a9bdcd984903381a94960f3885 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2023-09-29T06:15:18,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=A 2023-09-29T06:15:18,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:18,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=B 2023-09-29T06:15:18,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:18,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=C 2023-09-29T06:15:18,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:18,421 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/f00c2ed0b0ee45d4bca73ec4eb1e70f6 is 50, key is test_row_0/A:col10/1695968118416/Put/seqid=0 2023-09-29T06:15:18,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2023-09-29T06:15:18,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:18,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. as already flushing 2023-09-29T06:15:18,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:18,443 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(53): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:18,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:18,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:18,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:18,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968178453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:18,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:18,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968178454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:18,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:18,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968178454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:18,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:18,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:18,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968178459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:18,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968178460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:18,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:18,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968178561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:18,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:18,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968178561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:18,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:18,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968178561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:18,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:18,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968178565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:18,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:18,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968178565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:18,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=146 2023-09-29T06:15:18,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2023-09-29T06:15:18,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:18,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. as already flushing 2023-09-29T06:15:18,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:18,595 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(53): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:18,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:18,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:18,748 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2023-09-29T06:15:18,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:18,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. as already flushing 2023-09-29T06:15:18,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:18,748 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(53): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:18,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:18,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:18,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:18,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968178764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:18,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:18,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968178764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:18,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:18,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968178765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:18,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:18,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968178768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:18,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:18,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968178769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:18,825 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/f00c2ed0b0ee45d4bca73ec4eb1e70f6 2023-09-29T06:15:18,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/3778f02a488545d28b903df262458617 is 50, key is test_row_0/B:col10/1695968118416/Put/seqid=0 2023-09-29T06:15:18,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=146 2023-09-29T06:15:18,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2023-09-29T06:15:18,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:18,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. as already flushing 2023-09-29T06:15:18,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:18,901 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(53): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:18,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:18,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:19,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2023-09-29T06:15:19,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:19,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. as already flushing 2023-09-29T06:15:19,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:19,053 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(53): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:19,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:19,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:19,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:19,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968179068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:19,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:19,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968179069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:19,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:19,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968179070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:19,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:19,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968179073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:19,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:19,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968179074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:19,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2023-09-29T06:15:19,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:19,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. as already flushing 2023-09-29T06:15:19,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:19,206 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(53): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:19,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:19,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:19,235 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/3778f02a488545d28b903df262458617 2023-09-29T06:15:19,241 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/8eb47ec198f14026957170c698223235 is 50, key is test_row_0/C:col10/1695968118416/Put/seqid=0 2023-09-29T06:15:19,244 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/8eb47ec198f14026957170c698223235 2023-09-29T06:15:19,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/f00c2ed0b0ee45d4bca73ec4eb1e70f6 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/f00c2ed0b0ee45d4bca73ec4eb1e70f6 2023-09-29T06:15:19,251 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/f00c2ed0b0ee45d4bca73ec4eb1e70f6, entries=150, sequenceid=207, filesize=11.8 K 2023-09-29T06:15:19,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/3778f02a488545d28b903df262458617 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/3778f02a488545d28b903df262458617 2023-09-29T06:15:19,255 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/3778f02a488545d28b903df262458617, entries=150, sequenceid=207, filesize=11.8 K 2023-09-29T06:15:19,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/8eb47ec198f14026957170c698223235 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/8eb47ec198f14026957170c698223235 2023-09-29T06:15:19,258 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/8eb47ec198f14026957170c698223235, entries=150, sequenceid=207, filesize=11.8 K 2023-09-29T06:15:19,259 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 991924a9bdcd984903381a94960f3885 in 841ms, sequenceid=207, compaction requested=true 2023-09-29T06:15:19,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 991924a9bdcd984903381a94960f3885: 2023-09-29T06:15:19,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 991924a9bdcd984903381a94960f3885:A, priority=-2147483648, current under compaction store size is 1 2023-09-29T06:15:19,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:19,259 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:15:19,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 991924a9bdcd984903381a94960f3885:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:15:19,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:19,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 991924a9bdcd984903381a94960f3885:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:15:19,260 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:15:19,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:15:19,262 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36744 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:15:19,262 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 991924a9bdcd984903381a94960f3885/A is initiating minor compaction (all files) 2023-09-29T06:15:19,262 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36744 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:15:19,262 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 991924a9bdcd984903381a94960f3885/A in TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:19,262 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 991924a9bdcd984903381a94960f3885/B is initiating minor compaction (all files) 2023-09-29T06:15:19,262 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/e711c8235f31492eba6f558df55797b5, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/c5476dcba8e64f5fa402f88008b64ff3, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/f00c2ed0b0ee45d4bca73ec4eb1e70f6] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp, totalSize=35.9 K 2023-09-29T06:15:19,262 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 991924a9bdcd984903381a94960f3885/B in TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:19,262 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/f3bca8db01df42039df93f968704b5f8, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/959ca23e26c8448aafd0380ca8dc4406, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/3778f02a488545d28b903df262458617] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp, totalSize=35.9 K 2023-09-29T06:15:19,263 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting e711c8235f31492eba6f558df55797b5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1695968115812 2023-09-29T06:15:19,263 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting f3bca8db01df42039df93f968704b5f8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1695968115812 2023-09-29T06:15:19,263 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting c5476dcba8e64f5fa402f88008b64ff3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1695968116956 2023-09-29T06:15:19,263 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 959ca23e26c8448aafd0380ca8dc4406, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1695968116956 2023-09-29T06:15:19,263 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting f00c2ed0b0ee45d4bca73ec4eb1e70f6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1695968118097 2023-09-29T06:15:19,263 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 3778f02a488545d28b903df262458617, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1695968118097 2023-09-29T06:15:19,270 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 991924a9bdcd984903381a94960f3885#A#compaction#471 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:19,271 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 991924a9bdcd984903381a94960f3885#B#compaction#472 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:19,271 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/f13b9d58bb604a51ac1df67898c108aa is 50, key is test_row_0/A:col10/1695968118416/Put/seqid=0 2023-09-29T06:15:19,271 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/90e88b13ba984f5994a5d922c2bd636e is 50, key is test_row_0/B:col10/1695968118416/Put/seqid=0 2023-09-29T06:15:19,295 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/90e88b13ba984f5994a5d922c2bd636e as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/90e88b13ba984f5994a5d922c2bd636e 2023-09-29T06:15:19,299 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 991924a9bdcd984903381a94960f3885/B of 991924a9bdcd984903381a94960f3885 into 90e88b13ba984f5994a5d922c2bd636e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:19,299 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 991924a9bdcd984903381a94960f3885: 2023-09-29T06:15:19,299 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885., storeName=991924a9bdcd984903381a94960f3885/B, priority=13, startTime=1695968119259; duration=0sec 2023-09-29T06:15:19,299 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:15:19,299 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 991924a9bdcd984903381a94960f3885:B 2023-09-29T06:15:19,299 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:15:19,300 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36744 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:15:19,300 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 991924a9bdcd984903381a94960f3885/C is initiating minor compaction (all files) 2023-09-29T06:15:19,300 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 991924a9bdcd984903381a94960f3885/C in TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:19,300 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/8d927e6b7d72406ba1a62e38271543f4, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/e9ef77f047eb40f098a173c68a7f226f, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/8eb47ec198f14026957170c698223235] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp, totalSize=35.9 K 2023-09-29T06:15:19,301 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 8d927e6b7d72406ba1a62e38271543f4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1695968115812 2023-09-29T06:15:19,301 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting e9ef77f047eb40f098a173c68a7f226f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1695968116956 2023-09-29T06:15:19,301 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 8eb47ec198f14026957170c698223235, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1695968118097 2023-09-29T06:15:19,306 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 991924a9bdcd984903381a94960f3885#C#compaction#473 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:19,306 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/344d7841a89d42268b13c8854396e400 is 50, key is test_row_0/C:col10/1695968118416/Put/seqid=0 2023-09-29T06:15:19,313 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/344d7841a89d42268b13c8854396e400 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/344d7841a89d42268b13c8854396e400 2023-09-29T06:15:19,318 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 991924a9bdcd984903381a94960f3885/C of 991924a9bdcd984903381a94960f3885 into 344d7841a89d42268b13c8854396e400(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:19,318 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 991924a9bdcd984903381a94960f3885: 2023-09-29T06:15:19,318 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885., storeName=991924a9bdcd984903381a94960f3885/C, priority=13, startTime=1695968119259; duration=0sec 2023-09-29T06:15:19,318 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:19,318 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 991924a9bdcd984903381a94960f3885:C 2023-09-29T06:15:19,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2023-09-29T06:15:19,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:19,358 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2819): Flushing 991924a9bdcd984903381a94960f3885 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2023-09-29T06:15:19,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=A 2023-09-29T06:15:19,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:19,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=B 2023-09-29T06:15:19,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:19,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=C 2023-09-29T06:15:19,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:19,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/08c0b9df96d54196a01b0e3da179099a is 50, key is test_row_0/A:col10/1695968118459/Put/seqid=0 2023-09-29T06:15:19,365 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/08c0b9df96d54196a01b0e3da179099a 2023-09-29T06:15:19,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/bb3f7eebc94f4f7dad37ae74e74de7fc is 50, key is test_row_0/B:col10/1695968118459/Put/seqid=0 2023-09-29T06:15:19,373 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/bb3f7eebc94f4f7dad37ae74e74de7fc 2023-09-29T06:15:19,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/5bafcf91d1984f9c8b67d168def1ff3e is 50, key is test_row_0/C:col10/1695968118459/Put/seqid=0 2023-09-29T06:15:19,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=146 2023-09-29T06:15:19,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 991924a9bdcd984903381a94960f3885 2023-09-29T06:15:19,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. as already flushing 2023-09-29T06:15:19,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:19,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968179618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:19,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:19,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968179618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:19,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:19,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968179622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:19,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:19,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968179623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:19,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:19,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968179623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:19,688 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/f13b9d58bb604a51ac1df67898c108aa as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/f13b9d58bb604a51ac1df67898c108aa 2023-09-29T06:15:19,693 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 991924a9bdcd984903381a94960f3885/A of 991924a9bdcd984903381a94960f3885 into f13b9d58bb604a51ac1df67898c108aa(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:19,693 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 991924a9bdcd984903381a94960f3885: 2023-09-29T06:15:19,693 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885., storeName=991924a9bdcd984903381a94960f3885/A, priority=13, startTime=1695968119259; duration=0sec 2023-09-29T06:15:19,693 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:19,693 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 991924a9bdcd984903381a94960f3885:A 2023-09-29T06:15:19,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:19,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968179724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:19,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:19,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968179725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:19,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:19,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968179731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:19,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:19,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968179732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:19,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:19,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968179732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:19,782 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/5bafcf91d1984f9c8b67d168def1ff3e 2023-09-29T06:15:19,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/08c0b9df96d54196a01b0e3da179099a as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/08c0b9df96d54196a01b0e3da179099a 2023-09-29T06:15:19,789 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/08c0b9df96d54196a01b0e3da179099a, entries=150, sequenceid=225, filesize=11.8 K 2023-09-29T06:15:19,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/bb3f7eebc94f4f7dad37ae74e74de7fc as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/bb3f7eebc94f4f7dad37ae74e74de7fc 2023-09-29T06:15:19,793 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/bb3f7eebc94f4f7dad37ae74e74de7fc, entries=150, sequenceid=225, filesize=11.8 K 2023-09-29T06:15:19,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/5bafcf91d1984f9c8b67d168def1ff3e as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/5bafcf91d1984f9c8b67d168def1ff3e 2023-09-29T06:15:19,797 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/5bafcf91d1984f9c8b67d168def1ff3e, entries=150, sequenceid=225, filesize=11.8 K 2023-09-29T06:15:19,798 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3022): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for 991924a9bdcd984903381a94960f3885 in 439ms, sequenceid=225, compaction requested=false 2023-09-29T06:15:19,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2520): Flush status journal for 991924a9bdcd984903381a94960f3885: 2023-09-29T06:15:19,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:19,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=147 2023-09-29T06:15:19,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=147 2023-09-29T06:15:19,800 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=147, resume processing ppid=146 2023-09-29T06:15:19,800 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=147, ppid=146, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5090 sec 2023-09-29T06:15:19,801 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=146, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 1.5130 sec 2023-09-29T06:15:19,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 991924a9bdcd984903381a94960f3885 2023-09-29T06:15:19,932 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 991924a9bdcd984903381a94960f3885 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2023-09-29T06:15:19,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=A 2023-09-29T06:15:19,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:19,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=B 2023-09-29T06:15:19,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:19,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=C 2023-09-29T06:15:19,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:19,936 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/549c4d400f114c378d14a2d587f1eb01 is 50, key is test_row_0/A:col10/1695968119931/Put/seqid=0 2023-09-29T06:15:19,940 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/549c4d400f114c378d14a2d587f1eb01 2023-09-29T06:15:19,946 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/2ff8e04dbb184a8383275e812ea02efd is 50, key is test_row_0/B:col10/1695968119931/Put/seqid=0 2023-09-29T06:15:19,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:19,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968179944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:19,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:19,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968179946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:19,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:19,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968179947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:19,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:19,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968179948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:19,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:19,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968179948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:20,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:20,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968180052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:20,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:20,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968180053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:20,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:20,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968180053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:20,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:20,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968180053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:20,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:20,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968180056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:20,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:20,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968180255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:20,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:20,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968180257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:20,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:20,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968180257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:20,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:20,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968180257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:20,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:20,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968180259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:20,350 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/2ff8e04dbb184a8383275e812ea02efd 2023-09-29T06:15:20,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/18704b53e32b4f4987b2b9f61cefc12a is 50, key is test_row_0/C:col10/1695968119931/Put/seqid=0 2023-09-29T06:15:20,359 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/18704b53e32b4f4987b2b9f61cefc12a 2023-09-29T06:15:20,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/549c4d400f114c378d14a2d587f1eb01 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/549c4d400f114c378d14a2d587f1eb01 2023-09-29T06:15:20,366 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/549c4d400f114c378d14a2d587f1eb01, entries=200, sequenceid=249, filesize=14.2 K 2023-09-29T06:15:20,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/2ff8e04dbb184a8383275e812ea02efd as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/2ff8e04dbb184a8383275e812ea02efd 2023-09-29T06:15:20,370 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/2ff8e04dbb184a8383275e812ea02efd, entries=150, sequenceid=249, filesize=11.8 K 2023-09-29T06:15:20,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/18704b53e32b4f4987b2b9f61cefc12a as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/18704b53e32b4f4987b2b9f61cefc12a 2023-09-29T06:15:20,374 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/18704b53e32b4f4987b2b9f61cefc12a, entries=150, sequenceid=249, filesize=11.8 K 2023-09-29T06:15:20,375 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 991924a9bdcd984903381a94960f3885 in 443ms, sequenceid=249, compaction requested=true 2023-09-29T06:15:20,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 991924a9bdcd984903381a94960f3885: 2023-09-29T06:15:20,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 991924a9bdcd984903381a94960f3885:A, priority=-2147483648, current under compaction store size is 1 2023-09-29T06:15:20,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:20,375 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:15:20,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 991924a9bdcd984903381a94960f3885:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:15:20,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:20,375 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:15:20,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 991924a9bdcd984903381a94960f3885:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:15:20,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:15:20,376 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39236 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:15:20,376 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 991924a9bdcd984903381a94960f3885/A is initiating minor compaction (all files) 2023-09-29T06:15:20,376 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36846 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:15:20,376 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 991924a9bdcd984903381a94960f3885/B is initiating minor compaction (all files) 2023-09-29T06:15:20,376 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 991924a9bdcd984903381a94960f3885/A in TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:20,376 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 991924a9bdcd984903381a94960f3885/B in TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:20,376 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/90e88b13ba984f5994a5d922c2bd636e, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/bb3f7eebc94f4f7dad37ae74e74de7fc, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/2ff8e04dbb184a8383275e812ea02efd] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp, totalSize=36.0 K 2023-09-29T06:15:20,376 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/f13b9d58bb604a51ac1df67898c108aa, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/08c0b9df96d54196a01b0e3da179099a, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/549c4d400f114c378d14a2d587f1eb01] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp, totalSize=38.3 K 2023-09-29T06:15:20,377 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 90e88b13ba984f5994a5d922c2bd636e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1695968118097 2023-09-29T06:15:20,377 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting f13b9d58bb604a51ac1df67898c108aa, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1695968118097 2023-09-29T06:15:20,377 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting bb3f7eebc94f4f7dad37ae74e74de7fc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1695968118427 2023-09-29T06:15:20,377 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 08c0b9df96d54196a01b0e3da179099a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1695968118427 2023-09-29T06:15:20,377 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 549c4d400f114c378d14a2d587f1eb01, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1695968119616 2023-09-29T06:15:20,377 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 2ff8e04dbb184a8383275e812ea02efd, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1695968119616 2023-09-29T06:15:20,383 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 991924a9bdcd984903381a94960f3885#B#compaction#480 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:20,383 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/dfeeb62fd6c34a8f877f2fd7808bb410 is 50, key is test_row_0/B:col10/1695968119931/Put/seqid=0 2023-09-29T06:15:20,383 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 991924a9bdcd984903381a94960f3885#A#compaction#481 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:20,383 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/e3c8ad132ff64e4b93a89c74a9283154 is 50, key is test_row_0/A:col10/1695968119931/Put/seqid=0 2023-09-29T06:15:20,390 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/dfeeb62fd6c34a8f877f2fd7808bb410 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/dfeeb62fd6c34a8f877f2fd7808bb410 2023-09-29T06:15:20,393 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 991924a9bdcd984903381a94960f3885/B of 991924a9bdcd984903381a94960f3885 into dfeeb62fd6c34a8f877f2fd7808bb410(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:20,393 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 991924a9bdcd984903381a94960f3885: 2023-09-29T06:15:20,393 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885., storeName=991924a9bdcd984903381a94960f3885/B, priority=13, startTime=1695968120375; duration=0sec 2023-09-29T06:15:20,393 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:15:20,393 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 991924a9bdcd984903381a94960f3885:B 2023-09-29T06:15:20,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=146 2023-09-29T06:15:20,393 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:15:20,393 INFO [Thread-2713 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 146 completed 2023-09-29T06:15:20,395 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36846 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:15:20,395 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 991924a9bdcd984903381a94960f3885/C is initiating minor compaction (all files) 2023-09-29T06:15:20,395 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 991924a9bdcd984903381a94960f3885/C in TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:20,395 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/344d7841a89d42268b13c8854396e400, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/5bafcf91d1984f9c8b67d168def1ff3e, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/18704b53e32b4f4987b2b9f61cefc12a] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp, totalSize=36.0 K 2023-09-29T06:15:20,395 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 344d7841a89d42268b13c8854396e400, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1695968118097 2023-09-29T06:15:20,396 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:15:20,396 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 5bafcf91d1984f9c8b67d168def1ff3e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1695968118427 2023-09-29T06:15:20,396 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 18704b53e32b4f4987b2b9f61cefc12a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1695968119616 2023-09-29T06:15:20,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2023-09-29T06:15:20,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=148 2023-09-29T06:15:20,399 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:15:20,399 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:15:20,399 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:15:20,405 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/e3c8ad132ff64e4b93a89c74a9283154 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/e3c8ad132ff64e4b93a89c74a9283154 2023-09-29T06:15:20,411 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 991924a9bdcd984903381a94960f3885#C#compaction#482 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:20,412 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/8737ab3bdb2041bdb5e5ea2f087d3ee9 is 50, key is test_row_0/C:col10/1695968119931/Put/seqid=0 2023-09-29T06:15:20,412 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 991924a9bdcd984903381a94960f3885/A of 991924a9bdcd984903381a94960f3885 into e3c8ad132ff64e4b93a89c74a9283154(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:20,412 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 991924a9bdcd984903381a94960f3885: 2023-09-29T06:15:20,412 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885., storeName=991924a9bdcd984903381a94960f3885/A, priority=13, startTime=1695968120375; duration=0sec 2023-09-29T06:15:20,413 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:20,413 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 991924a9bdcd984903381a94960f3885:A 2023-09-29T06:15:20,441 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/8737ab3bdb2041bdb5e5ea2f087d3ee9 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/8737ab3bdb2041bdb5e5ea2f087d3ee9 2023-09-29T06:15:20,445 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 991924a9bdcd984903381a94960f3885/C of 991924a9bdcd984903381a94960f3885 into 8737ab3bdb2041bdb5e5ea2f087d3ee9(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:20,445 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 991924a9bdcd984903381a94960f3885: 2023-09-29T06:15:20,445 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885., storeName=991924a9bdcd984903381a94960f3885/C, priority=13, startTime=1695968120375; duration=0sec 2023-09-29T06:15:20,445 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:20,445 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 991924a9bdcd984903381a94960f3885:C 2023-09-29T06:15:20,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=148 2023-09-29T06:15:20,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2023-09-29T06:15:20,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:20,553 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2819): Flushing 991924a9bdcd984903381a94960f3885 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2023-09-29T06:15:20,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=A 2023-09-29T06:15:20,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:20,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=B 2023-09-29T06:15:20,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:20,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=C 2023-09-29T06:15:20,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:20,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/117076fcfd1c482f8feedfb9e6ba8318 is 50, key is test_row_0/A:col10/1695968119938/Put/seqid=0 2023-09-29T06:15:20,562 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/117076fcfd1c482f8feedfb9e6ba8318 2023-09-29T06:15:20,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. as already flushing 2023-09-29T06:15:20,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 991924a9bdcd984903381a94960f3885 2023-09-29T06:15:20,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/3d9850737c604a89bfba4156c893f683 is 50, key is test_row_0/B:col10/1695968119938/Put/seqid=0 2023-09-29T06:15:20,572 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/3d9850737c604a89bfba4156c893f683 2023-09-29T06:15:20,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/7609c6376fb944c5801da3acb2dc3b68 is 50, key is test_row_0/C:col10/1695968119938/Put/seqid=0 2023-09-29T06:15:20,586 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/7609c6376fb944c5801da3acb2dc3b68 2023-09-29T06:15:20,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/117076fcfd1c482f8feedfb9e6ba8318 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/117076fcfd1c482f8feedfb9e6ba8318 2023-09-29T06:15:20,591 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/117076fcfd1c482f8feedfb9e6ba8318, entries=150, sequenceid=266, filesize=11.9 K 2023-09-29T06:15:20,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/3d9850737c604a89bfba4156c893f683 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/3d9850737c604a89bfba4156c893f683 2023-09-29T06:15:20,595 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/3d9850737c604a89bfba4156c893f683, entries=150, sequenceid=266, filesize=11.9 K 2023-09-29T06:15:20,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/7609c6376fb944c5801da3acb2dc3b68 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/7609c6376fb944c5801da3acb2dc3b68 2023-09-29T06:15:20,598 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/7609c6376fb944c5801da3acb2dc3b68, entries=150, sequenceid=266, filesize=11.9 K 2023-09-29T06:15:20,605 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3022): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=87.22 KB/89310 for 991924a9bdcd984903381a94960f3885 in 51ms, sequenceid=266, compaction requested=false 2023-09-29T06:15:20,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2520): Flush status journal for 991924a9bdcd984903381a94960f3885: 2023-09-29T06:15:20,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:20,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=149 2023-09-29T06:15:20,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=149 2023-09-29T06:15:20,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 991924a9bdcd984903381a94960f3885 2023-09-29T06:15:20,605 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 991924a9bdcd984903381a94960f3885 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2023-09-29T06:15:20,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=A 2023-09-29T06:15:20,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:20,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=B 2023-09-29T06:15:20,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:20,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=C 2023-09-29T06:15:20,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:20,607 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=149, resume processing ppid=148 2023-09-29T06:15:20,607 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=149, ppid=148, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 207 msec 2023-09-29T06:15:20,609 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=148, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 212 msec 2023-09-29T06:15:20,610 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/4c30970feee7484ca1e5b69adaf87fa3 is 50, key is test_row_0/A:col10/1695968120580/Put/seqid=0 2023-09-29T06:15:20,614 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/4c30970feee7484ca1e5b69adaf87fa3 2023-09-29T06:15:20,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/07ec4d1681e64d80b3abf6fe3ba3834c is 50, key is test_row_0/B:col10/1695968120580/Put/seqid=0 2023-09-29T06:15:20,622 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/07ec4d1681e64d80b3abf6fe3ba3834c 2023-09-29T06:15:20,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:20,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968180626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:20,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/24e3ea0bbf1b45c2b1557a800ad4a8db is 50, key is test_row_0/C:col10/1695968120580/Put/seqid=0 2023-09-29T06:15:20,633 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/24e3ea0bbf1b45c2b1557a800ad4a8db 2023-09-29T06:15:20,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:20,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968180626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:20,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:20,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968180627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:20,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:20,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968180627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:20,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:20,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968180628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:20,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/4c30970feee7484ca1e5b69adaf87fa3 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/4c30970feee7484ca1e5b69adaf87fa3 2023-09-29T06:15:20,649 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/4c30970feee7484ca1e5b69adaf87fa3, entries=150, sequenceid=285, filesize=12.0 K 2023-09-29T06:15:20,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/07ec4d1681e64d80b3abf6fe3ba3834c as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/07ec4d1681e64d80b3abf6fe3ba3834c 2023-09-29T06:15:20,653 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/07ec4d1681e64d80b3abf6fe3ba3834c, entries=150, sequenceid=285, filesize=12.0 K 2023-09-29T06:15:20,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/24e3ea0bbf1b45c2b1557a800ad4a8db as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/24e3ea0bbf1b45c2b1557a800ad4a8db 2023-09-29T06:15:20,657 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/24e3ea0bbf1b45c2b1557a800ad4a8db, entries=150, sequenceid=285, filesize=12.0 K 2023-09-29T06:15:20,658 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 991924a9bdcd984903381a94960f3885 in 53ms, sequenceid=285, compaction requested=true 2023-09-29T06:15:20,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 991924a9bdcd984903381a94960f3885: 2023-09-29T06:15:20,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 991924a9bdcd984903381a94960f3885:A, priority=-2147483648, current under compaction store size is 1 2023-09-29T06:15:20,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:20,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 991924a9bdcd984903381a94960f3885:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:15:20,658 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:15:20,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:20,658 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:15:20,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 991924a9bdcd984903381a94960f3885:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:15:20,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:15:20,659 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37198 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:15:20,659 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 991924a9bdcd984903381a94960f3885/A is initiating minor compaction (all files) 2023-09-29T06:15:20,659 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37198 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:15:20,659 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 991924a9bdcd984903381a94960f3885/A in TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:20,659 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 991924a9bdcd984903381a94960f3885/B is initiating minor compaction (all files) 2023-09-29T06:15:20,659 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 991924a9bdcd984903381a94960f3885/B in TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:20,659 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/e3c8ad132ff64e4b93a89c74a9283154, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/117076fcfd1c482f8feedfb9e6ba8318, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/4c30970feee7484ca1e5b69adaf87fa3] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp, totalSize=36.3 K 2023-09-29T06:15:20,659 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/dfeeb62fd6c34a8f877f2fd7808bb410, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/3d9850737c604a89bfba4156c893f683, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/07ec4d1681e64d80b3abf6fe3ba3834c] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp, totalSize=36.3 K 2023-09-29T06:15:20,660 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting e3c8ad132ff64e4b93a89c74a9283154, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1695968119616 2023-09-29T06:15:20,660 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting dfeeb62fd6c34a8f877f2fd7808bb410, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1695968119616 2023-09-29T06:15:20,660 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 117076fcfd1c482f8feedfb9e6ba8318, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1695968119938 2023-09-29T06:15:20,660 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 3d9850737c604a89bfba4156c893f683, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1695968119938 2023-09-29T06:15:20,660 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 4c30970feee7484ca1e5b69adaf87fa3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1695968120576 2023-09-29T06:15:20,661 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 07ec4d1681e64d80b3abf6fe3ba3834c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1695968120576 2023-09-29T06:15:20,666 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 991924a9bdcd984903381a94960f3885#A#compaction#489 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:20,666 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 991924a9bdcd984903381a94960f3885#B#compaction#490 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:20,667 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/74fc72851b7e4abba5d7ba3e226013e0 is 50, key is test_row_0/A:col10/1695968120580/Put/seqid=0 2023-09-29T06:15:20,667 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/8cc3305265af4a68940b418879baa53c is 50, key is test_row_0/B:col10/1695968120580/Put/seqid=0 2023-09-29T06:15:20,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=148 2023-09-29T06:15:20,700 INFO [Thread-2713 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 148 completed 2023-09-29T06:15:20,702 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:15:20,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2023-09-29T06:15:20,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=150 2023-09-29T06:15:20,704 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:15:20,705 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:15:20,705 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:15:20,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 991924a9bdcd984903381a94960f3885 2023-09-29T06:15:20,731 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 991924a9bdcd984903381a94960f3885 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2023-09-29T06:15:20,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=A 2023-09-29T06:15:20,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:20,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=B 2023-09-29T06:15:20,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:20,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=C 2023-09-29T06:15:20,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:20,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/3628c3be964a4d55879b376f03063fdd is 50, key is test_row_0/A:col10/1695968120730/Put/seqid=0 2023-09-29T06:15:20,740 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/3628c3be964a4d55879b376f03063fdd 2023-09-29T06:15:20,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/c4bf4767526f459ea6eea1d609dce1e2 is 50, key is test_row_0/B:col10/1695968120730/Put/seqid=0 2023-09-29T06:15:20,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:20,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968180755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:20,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:20,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968180755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:20,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:20,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968180758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:20,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:20,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968180760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:20,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:20,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968180761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:20,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=150 2023-09-29T06:15:20,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2023-09-29T06:15:20,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:20,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. as already flushing 2023-09-29T06:15:20,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:20,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(53): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:20,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:20,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:20,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:20,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968180862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:20,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:20,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968180862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:20,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:20,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968180862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:20,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:20,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968180864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:20,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:20,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968180867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:21,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=150 2023-09-29T06:15:21,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2023-09-29T06:15:21,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:21,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. as already flushing 2023-09-29T06:15:21,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:21,010 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(53): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:21,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:21,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:21,075 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/8cc3305265af4a68940b418879baa53c as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/8cc3305265af4a68940b418879baa53c 2023-09-29T06:15:21,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:21,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968181068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:21,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:21,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968181068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:21,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:21,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968181068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:21,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:21,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968181069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:21,076 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/74fc72851b7e4abba5d7ba3e226013e0 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/74fc72851b7e4abba5d7ba3e226013e0 2023-09-29T06:15:21,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:21,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968181073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:21,080 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 991924a9bdcd984903381a94960f3885/B of 991924a9bdcd984903381a94960f3885 into 8cc3305265af4a68940b418879baa53c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:21,080 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 991924a9bdcd984903381a94960f3885: 2023-09-29T06:15:21,080 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885., storeName=991924a9bdcd984903381a94960f3885/B, priority=13, startTime=1695968120658; duration=0sec 2023-09-29T06:15:21,080 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:15:21,080 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 991924a9bdcd984903381a94960f3885:B 2023-09-29T06:15:21,080 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:15:21,080 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 991924a9bdcd984903381a94960f3885/A of 991924a9bdcd984903381a94960f3885 into 74fc72851b7e4abba5d7ba3e226013e0(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:21,080 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 991924a9bdcd984903381a94960f3885: 2023-09-29T06:15:21,080 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885., storeName=991924a9bdcd984903381a94960f3885/A, priority=13, startTime=1695968120658; duration=0sec 2023-09-29T06:15:21,080 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:21,080 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 991924a9bdcd984903381a94960f3885:A 2023-09-29T06:15:21,081 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37198 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:15:21,081 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 991924a9bdcd984903381a94960f3885/C is initiating minor compaction (all files) 2023-09-29T06:15:21,081 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 991924a9bdcd984903381a94960f3885/C in TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:21,081 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/8737ab3bdb2041bdb5e5ea2f087d3ee9, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/7609c6376fb944c5801da3acb2dc3b68, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/24e3ea0bbf1b45c2b1557a800ad4a8db] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp, totalSize=36.3 K 2023-09-29T06:15:21,082 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 8737ab3bdb2041bdb5e5ea2f087d3ee9, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1695968119616 2023-09-29T06:15:21,082 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 7609c6376fb944c5801da3acb2dc3b68, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1695968119938 2023-09-29T06:15:21,082 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 24e3ea0bbf1b45c2b1557a800ad4a8db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1695968120576 2023-09-29T06:15:21,088 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 991924a9bdcd984903381a94960f3885#C#compaction#493 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:21,088 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/b1fcb6cdbdb548b68484c29645009b5a is 50, key is test_row_0/C:col10/1695968120580/Put/seqid=0 2023-09-29T06:15:21,149 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/c4bf4767526f459ea6eea1d609dce1e2 2023-09-29T06:15:21,155 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/55a6b35eac8d4fabb0128cf0a9a96714 is 50, key is test_row_0/C:col10/1695968120730/Put/seqid=0 2023-09-29T06:15:21,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2023-09-29T06:15:21,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:21,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. as already flushing 2023-09-29T06:15:21,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:21,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(53): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:21,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:21,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:21,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=150 2023-09-29T06:15:21,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2023-09-29T06:15:21,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:21,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. as already flushing 2023-09-29T06:15:21,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:21,315 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(53): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:21,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:21,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:21,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:21,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968181376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:21,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:21,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968181377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:21,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:21,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968181377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:21,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:21,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968181377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:21,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:21,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968181380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:21,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2023-09-29T06:15:21,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:21,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. as already flushing 2023-09-29T06:15:21,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:21,468 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(53): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:21,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:21,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:21,495 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/b1fcb6cdbdb548b68484c29645009b5a as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/b1fcb6cdbdb548b68484c29645009b5a 2023-09-29T06:15:21,508 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 991924a9bdcd984903381a94960f3885/C of 991924a9bdcd984903381a94960f3885 into b1fcb6cdbdb548b68484c29645009b5a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:21,508 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 991924a9bdcd984903381a94960f3885: 2023-09-29T06:15:21,508 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885., storeName=991924a9bdcd984903381a94960f3885/C, priority=13, startTime=1695968120658; duration=0sec 2023-09-29T06:15:21,508 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:21,508 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 991924a9bdcd984903381a94960f3885:C 2023-09-29T06:15:21,559 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/55a6b35eac8d4fabb0128cf0a9a96714 2023-09-29T06:15:21,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/3628c3be964a4d55879b376f03063fdd as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/3628c3be964a4d55879b376f03063fdd 2023-09-29T06:15:21,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/3628c3be964a4d55879b376f03063fdd, entries=200, sequenceid=303, filesize=14.4 K 2023-09-29T06:15:21,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/c4bf4767526f459ea6eea1d609dce1e2 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/c4bf4767526f459ea6eea1d609dce1e2 2023-09-29T06:15:21,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/c4bf4767526f459ea6eea1d609dce1e2, entries=150, sequenceid=303, filesize=12.0 K 2023-09-29T06:15:21,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/55a6b35eac8d4fabb0128cf0a9a96714 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/55a6b35eac8d4fabb0128cf0a9a96714 2023-09-29T06:15:21,572 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/55a6b35eac8d4fabb0128cf0a9a96714, entries=150, sequenceid=303, filesize=12.0 K 2023-09-29T06:15:21,573 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for 991924a9bdcd984903381a94960f3885 in 842ms, sequenceid=303, compaction requested=false 2023-09-29T06:15:21,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 991924a9bdcd984903381a94960f3885: 2023-09-29T06:15:21,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2023-09-29T06:15:21,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:21,620 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2819): Flushing 991924a9bdcd984903381a94960f3885 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2023-09-29T06:15:21,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=A 2023-09-29T06:15:21,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:21,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=B 2023-09-29T06:15:21,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:21,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=C 2023-09-29T06:15:21,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:21,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/91f6a2a652af4d058b91ac6687ab79e6 is 50, key is test_row_0/A:col10/1695968120760/Put/seqid=0 2023-09-29T06:15:21,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=150 2023-09-29T06:15:21,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 991924a9bdcd984903381a94960f3885 2023-09-29T06:15:21,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. as already flushing 2023-09-29T06:15:21,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:21,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968181893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:21,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:21,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968181895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:21,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:21,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968181896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:21,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:21,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968181897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:21,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:21,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968181898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:22,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:22,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968181999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:22,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:22,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968182000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:22,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:22,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968182000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:22,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:22,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968182003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:22,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:22,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968182003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:22,028 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/91f6a2a652af4d058b91ac6687ab79e6 2023-09-29T06:15:22,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/660288b0de59436dbe0c7f75a8d19927 is 50, key is test_row_0/B:col10/1695968120760/Put/seqid=0 2023-09-29T06:15:22,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:22,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968182204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:22,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:22,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968182204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:22,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:22,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968182204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:22,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:22,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968182209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:22,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:22,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968182209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:22,438 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/660288b0de59436dbe0c7f75a8d19927 2023-09-29T06:15:22,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/f52a47f4740548428be9961164c63f64 is 50, key is test_row_0/C:col10/1695968120760/Put/seqid=0 2023-09-29T06:15:22,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:22,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968182508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:22,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:22,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968182509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:22,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:22,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968182509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:22,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:22,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968182514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:22,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:22,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968182515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:22,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=150 2023-09-29T06:15:22,851 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/f52a47f4740548428be9961164c63f64 2023-09-29T06:15:22,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/91f6a2a652af4d058b91ac6687ab79e6 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/91f6a2a652af4d058b91ac6687ab79e6 2023-09-29T06:15:22,858 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/91f6a2a652af4d058b91ac6687ab79e6, entries=150, sequenceid=325, filesize=12.0 K 2023-09-29T06:15:22,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/660288b0de59436dbe0c7f75a8d19927 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/660288b0de59436dbe0c7f75a8d19927 2023-09-29T06:15:22,862 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/660288b0de59436dbe0c7f75a8d19927, entries=150, sequenceid=325, filesize=12.0 K 2023-09-29T06:15:22,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/f52a47f4740548428be9961164c63f64 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/f52a47f4740548428be9961164c63f64 2023-09-29T06:15:22,866 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/f52a47f4740548428be9961164c63f64, entries=150, sequenceid=325, filesize=12.0 K 2023-09-29T06:15:22,867 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3022): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 991924a9bdcd984903381a94960f3885 in 1247ms, sequenceid=325, compaction requested=true 2023-09-29T06:15:22,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2520): Flush status journal for 991924a9bdcd984903381a94960f3885: 2023-09-29T06:15:22,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:22,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=151 2023-09-29T06:15:22,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=151 2023-09-29T06:15:22,869 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=151, resume processing ppid=150 2023-09-29T06:15:22,869 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=151, ppid=150, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1630 sec 2023-09-29T06:15:22,870 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=150, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 2.1670 sec 2023-09-29T06:15:23,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 991924a9bdcd984903381a94960f3885 2023-09-29T06:15:23,014 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 991924a9bdcd984903381a94960f3885 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2023-09-29T06:15:23,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=A 2023-09-29T06:15:23,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:23,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=B 2023-09-29T06:15:23,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:23,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=C 2023-09-29T06:15:23,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:23,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/a1d47f74f1c2422899fa5a3cc919b4d2 is 50, key is test_row_0/A:col10/1695968121896/Put/seqid=0 2023-09-29T06:15:23,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:23,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968183030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:23,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:23,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968183031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:23,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:23,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968183035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:23,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:23,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968183035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:23,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:23,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968183036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:23,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:23,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968183140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:23,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:23,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968183140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:23,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:23,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968183141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:23,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:23,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968183141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:23,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:23,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968183142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:23,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:23,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968183344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:23,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:23,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968183345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:23,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:23,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968183345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:23,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:23,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968183346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:23,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:23,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968183346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:23,423 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/a1d47f74f1c2422899fa5a3cc919b4d2 2023-09-29T06:15:23,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/d4122ffd630c47a9b3c291ec47b6a00b is 50, key is test_row_0/B:col10/1695968121896/Put/seqid=0 2023-09-29T06:15:23,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:23,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968183648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:23,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:23,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968183650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:23,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:23,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968183651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:23,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:23,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968183651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:23,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:23,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968183652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:23,833 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/d4122ffd630c47a9b3c291ec47b6a00b 2023-09-29T06:15:23,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/4159035352a645e3a627850e05b5a251 is 50, key is test_row_0/C:col10/1695968121896/Put/seqid=0 2023-09-29T06:15:24,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:24,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968184153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:24,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:24,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968184156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:24,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:24,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968184157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:24,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:24,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968184158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:24,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:24,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968184158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:24,243 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/4159035352a645e3a627850e05b5a251 2023-09-29T06:15:24,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/a1d47f74f1c2422899fa5a3cc919b4d2 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/a1d47f74f1c2422899fa5a3cc919b4d2 2023-09-29T06:15:24,250 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/a1d47f74f1c2422899fa5a3cc919b4d2, entries=200, sequenceid=344, filesize=14.4 K 2023-09-29T06:15:24,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/d4122ffd630c47a9b3c291ec47b6a00b as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/d4122ffd630c47a9b3c291ec47b6a00b 2023-09-29T06:15:24,254 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/d4122ffd630c47a9b3c291ec47b6a00b, entries=150, sequenceid=344, filesize=12.0 K 2023-09-29T06:15:24,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/4159035352a645e3a627850e05b5a251 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/4159035352a645e3a627850e05b5a251 2023-09-29T06:15:24,258 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/4159035352a645e3a627850e05b5a251, entries=150, sequenceid=344, filesize=12.0 K 2023-09-29T06:15:24,258 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 991924a9bdcd984903381a94960f3885 in 1244ms, sequenceid=344, compaction requested=true 2023-09-29T06:15:24,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 991924a9bdcd984903381a94960f3885: 2023-09-29T06:15:24,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 991924a9bdcd984903381a94960f3885:A, priority=-2147483648, current under compaction store size is 1 2023-09-29T06:15:24,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:24,259 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-29T06:15:24,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 991924a9bdcd984903381a94960f3885:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:15:24,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:24,259 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-29T06:15:24,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 991924a9bdcd984903381a94960f3885:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:15:24,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:15:24,260 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54664 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-29T06:15:24,260 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 991924a9bdcd984903381a94960f3885/A is initiating minor compaction (all files) 2023-09-29T06:15:24,260 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 991924a9bdcd984903381a94960f3885/A in TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:24,260 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/74fc72851b7e4abba5d7ba3e226013e0, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/3628c3be964a4d55879b376f03063fdd, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/91f6a2a652af4d058b91ac6687ab79e6, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/a1d47f74f1c2422899fa5a3cc919b4d2] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp, totalSize=53.4 K 2023-09-29T06:15:24,260 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-29T06:15:24,260 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 991924a9bdcd984903381a94960f3885/B is initiating minor compaction (all files) 2023-09-29T06:15:24,260 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 991924a9bdcd984903381a94960f3885/B in TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:24,260 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/8cc3305265af4a68940b418879baa53c, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/c4bf4767526f459ea6eea1d609dce1e2, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/660288b0de59436dbe0c7f75a8d19927, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/d4122ffd630c47a9b3c291ec47b6a00b] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp, totalSize=48.6 K 2023-09-29T06:15:24,260 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 74fc72851b7e4abba5d7ba3e226013e0, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1695968120576 2023-09-29T06:15:24,261 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 8cc3305265af4a68940b418879baa53c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1695968120576 2023-09-29T06:15:24,261 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 3628c3be964a4d55879b376f03063fdd, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1695968120625 2023-09-29T06:15:24,261 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting c4bf4767526f459ea6eea1d609dce1e2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1695968120625 2023-09-29T06:15:24,261 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 91f6a2a652af4d058b91ac6687ab79e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1695968120750 2023-09-29T06:15:24,261 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 660288b0de59436dbe0c7f75a8d19927, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1695968120750 2023-09-29T06:15:24,262 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting a1d47f74f1c2422899fa5a3cc919b4d2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1695968121896 2023-09-29T06:15:24,262 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting d4122ffd630c47a9b3c291ec47b6a00b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1695968121896 2023-09-29T06:15:24,269 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 991924a9bdcd984903381a94960f3885#A#compaction#501 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:24,270 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 991924a9bdcd984903381a94960f3885#B#compaction#502 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:24,270 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/1f21f563ba98477399fbab72a2822249 is 50, key is test_row_0/A:col10/1695968121896/Put/seqid=0 2023-09-29T06:15:24,270 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/f813ceda39774924875c97a64b342258 is 50, key is test_row_0/B:col10/1695968121896/Put/seqid=0 2023-09-29T06:15:24,281 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/1f21f563ba98477399fbab72a2822249 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/1f21f563ba98477399fbab72a2822249 2023-09-29T06:15:24,281 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/f813ceda39774924875c97a64b342258 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/f813ceda39774924875c97a64b342258 2023-09-29T06:15:24,287 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 991924a9bdcd984903381a94960f3885/A of 991924a9bdcd984903381a94960f3885 into 1f21f563ba98477399fbab72a2822249(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:24,287 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 991924a9bdcd984903381a94960f3885: 2023-09-29T06:15:24,287 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885., storeName=991924a9bdcd984903381a94960f3885/A, priority=12, startTime=1695968124259; duration=0sec 2023-09-29T06:15:24,287 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 991924a9bdcd984903381a94960f3885/B of 991924a9bdcd984903381a94960f3885 into f813ceda39774924875c97a64b342258(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:24,287 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:15:24,287 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 991924a9bdcd984903381a94960f3885: 2023-09-29T06:15:24,287 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 991924a9bdcd984903381a94960f3885:A 2023-09-29T06:15:24,287 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885., storeName=991924a9bdcd984903381a94960f3885/B, priority=12, startTime=1695968124259; duration=0sec 2023-09-29T06:15:24,287 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:24,287 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 991924a9bdcd984903381a94960f3885:B 2023-09-29T06:15:24,287 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-29T06:15:24,289 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-29T06:15:24,289 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 991924a9bdcd984903381a94960f3885/C is initiating minor compaction (all files) 2023-09-29T06:15:24,289 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 991924a9bdcd984903381a94960f3885/C in TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:24,289 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/b1fcb6cdbdb548b68484c29645009b5a, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/55a6b35eac8d4fabb0128cf0a9a96714, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/f52a47f4740548428be9961164c63f64, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/4159035352a645e3a627850e05b5a251] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp, totalSize=48.6 K 2023-09-29T06:15:24,290 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting b1fcb6cdbdb548b68484c29645009b5a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1695968120576 2023-09-29T06:15:24,291 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 55a6b35eac8d4fabb0128cf0a9a96714, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1695968120625 2023-09-29T06:15:24,292 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting f52a47f4740548428be9961164c63f64, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1695968120750 2023-09-29T06:15:24,292 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 4159035352a645e3a627850e05b5a251, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1695968121896 2023-09-29T06:15:24,299 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 991924a9bdcd984903381a94960f3885#C#compaction#503 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:24,299 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/9cb0431f12f64365b1300340aac54b4c is 50, key is test_row_0/C:col10/1695968121896/Put/seqid=0 2023-09-29T06:15:24,709 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/9cb0431f12f64365b1300340aac54b4c as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/9cb0431f12f64365b1300340aac54b4c 2023-09-29T06:15:24,713 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 991924a9bdcd984903381a94960f3885/C of 991924a9bdcd984903381a94960f3885 into 9cb0431f12f64365b1300340aac54b4c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:24,713 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 991924a9bdcd984903381a94960f3885: 2023-09-29T06:15:24,713 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885., storeName=991924a9bdcd984903381a94960f3885/C, priority=12, startTime=1695968124259; duration=0sec 2023-09-29T06:15:24,713 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:24,713 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 991924a9bdcd984903381a94960f3885:C 2023-09-29T06:15:24,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=150 2023-09-29T06:15:24,809 INFO [Thread-2713 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 150 completed 2023-09-29T06:15:24,811 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:15:24,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2023-09-29T06:15:24,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=152 2023-09-29T06:15:24,813 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:15:24,814 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:15:24,814 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:15:24,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=152 2023-09-29T06:15:24,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2023-09-29T06:15:24,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:24,967 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2819): Flushing 991924a9bdcd984903381a94960f3885 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2023-09-29T06:15:24,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=A 2023-09-29T06:15:24,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:24,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=B 2023-09-29T06:15:24,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:24,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=C 2023-09-29T06:15:24,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:24,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/6a635bae6dba4a5f82392090d466bd5c is 50, key is test_row_0/A:col10/1695968123034/Put/seqid=0 2023-09-29T06:15:25,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=152 2023-09-29T06:15:25,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 991924a9bdcd984903381a94960f3885 2023-09-29T06:15:25,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. as already flushing 2023-09-29T06:15:25,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:25,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968185177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:25,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:25,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968185178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:25,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:25,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968185179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:25,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:25,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968185179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:25,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:25,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968185180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:25,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:25,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968185282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:25,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:25,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968185283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:25,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:25,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968185284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:25,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:25,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968185284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:25,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:25,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968185284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:25,376 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/6a635bae6dba4a5f82392090d466bd5c 2023-09-29T06:15:25,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/2ee5669aa9344b6da7efb6c51668eae6 is 50, key is test_row_0/B:col10/1695968123034/Put/seqid=0 2023-09-29T06:15:25,386 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/2ee5669aa9344b6da7efb6c51668eae6 2023-09-29T06:15:25,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/9ca0c6a5f5b74672bd41df8bbd4b89e0 is 50, key is test_row_0/C:col10/1695968123034/Put/seqid=0 2023-09-29T06:15:25,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=152 2023-09-29T06:15:25,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:25,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968185485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:25,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:25,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968185485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:25,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:25,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968185487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:25,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:25,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968185487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:25,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:25,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968185489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:25,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:25,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968185788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:25,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:25,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968185789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:25,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:25,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968185790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:25,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:25,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968185791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:25,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:25,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968185792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:25,799 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/9ca0c6a5f5b74672bd41df8bbd4b89e0 2023-09-29T06:15:25,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/6a635bae6dba4a5f82392090d466bd5c as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/6a635bae6dba4a5f82392090d466bd5c 2023-09-29T06:15:25,807 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/6a635bae6dba4a5f82392090d466bd5c, entries=150, sequenceid=364, filesize=12.0 K 2023-09-29T06:15:25,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/2ee5669aa9344b6da7efb6c51668eae6 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/2ee5669aa9344b6da7efb6c51668eae6 2023-09-29T06:15:25,811 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/2ee5669aa9344b6da7efb6c51668eae6, entries=150, sequenceid=364, filesize=12.0 K 2023-09-29T06:15:25,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/9ca0c6a5f5b74672bd41df8bbd4b89e0 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/9ca0c6a5f5b74672bd41df8bbd4b89e0 2023-09-29T06:15:25,815 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/9ca0c6a5f5b74672bd41df8bbd4b89e0, entries=150, sequenceid=364, filesize=12.0 K 2023-09-29T06:15:25,816 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3022): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 991924a9bdcd984903381a94960f3885 in 849ms, sequenceid=364, compaction requested=false 2023-09-29T06:15:25,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2520): Flush status journal for 991924a9bdcd984903381a94960f3885: 2023-09-29T06:15:25,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:25,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=153 2023-09-29T06:15:25,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=153 2023-09-29T06:15:25,818 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=153, resume processing ppid=152 2023-09-29T06:15:25,818 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=153, ppid=152, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0030 sec 2023-09-29T06:15:25,820 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=152, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.0080 sec 2023-09-29T06:15:25,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=152 2023-09-29T06:15:25,919 INFO [Thread-2713 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 152 completed 2023-09-29T06:15:25,920 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:15:25,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2023-09-29T06:15:25,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=154 2023-09-29T06:15:25,923 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:15:25,923 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:15:25,923 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:15:26,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=154 2023-09-29T06:15:26,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2023-09-29T06:15:26,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:26,076 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2819): Flushing 991924a9bdcd984903381a94960f3885 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2023-09-29T06:15:26,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=A 2023-09-29T06:15:26,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:26,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=B 2023-09-29T06:15:26,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:26,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=C 2023-09-29T06:15:26,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:26,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/ded1b9399eb2459ba6f6edcadc9bdc10 is 50, key is test_row_0/A:col10/1695968125179/Put/seqid=0 2023-09-29T06:15:26,085 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/ded1b9399eb2459ba6f6edcadc9bdc10 2023-09-29T06:15:26,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/553d4f0ddda14276b0f002fc58031ba5 is 50, key is test_row_0/B:col10/1695968125179/Put/seqid=0 2023-09-29T06:15:26,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=154 2023-09-29T06:15:26,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 991924a9bdcd984903381a94960f3885 2023-09-29T06:15:26,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. as already flushing 2023-09-29T06:15:26,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:26,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968186310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:26,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:26,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968186310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:26,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:26,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968186311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:26,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:26,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968186312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:26,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:26,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968186313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:26,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:26,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968186419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:26,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:26,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968186419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:26,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:26,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968186419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:26,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:26,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968186420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:26,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:26,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968186420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:26,495 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/553d4f0ddda14276b0f002fc58031ba5 2023-09-29T06:15:26,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/a3956bffcc6545db876c7ec705ad7090 is 50, key is test_row_0/C:col10/1695968125179/Put/seqid=0 2023-09-29T06:15:26,504 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/a3956bffcc6545db876c7ec705ad7090 2023-09-29T06:15:26,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/ded1b9399eb2459ba6f6edcadc9bdc10 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/ded1b9399eb2459ba6f6edcadc9bdc10 2023-09-29T06:15:26,511 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/ded1b9399eb2459ba6f6edcadc9bdc10, entries=150, sequenceid=383, filesize=12.0 K 2023-09-29T06:15:26,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/553d4f0ddda14276b0f002fc58031ba5 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/553d4f0ddda14276b0f002fc58031ba5 2023-09-29T06:15:26,515 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/553d4f0ddda14276b0f002fc58031ba5, entries=150, sequenceid=383, filesize=12.0 K 2023-09-29T06:15:26,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/a3956bffcc6545db876c7ec705ad7090 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/a3956bffcc6545db876c7ec705ad7090 2023-09-29T06:15:26,519 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/a3956bffcc6545db876c7ec705ad7090, entries=150, sequenceid=383, filesize=12.0 K 2023-09-29T06:15:26,520 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3022): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 991924a9bdcd984903381a94960f3885 in 444ms, sequenceid=383, compaction requested=true 2023-09-29T06:15:26,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2520): Flush status journal for 991924a9bdcd984903381a94960f3885: 2023-09-29T06:15:26,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:26,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=155 2023-09-29T06:15:26,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=155 2023-09-29T06:15:26,522 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=155, resume processing ppid=154 2023-09-29T06:15:26,522 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=155, ppid=154, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 598 msec 2023-09-29T06:15:26,523 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=154, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 602 msec 2023-09-29T06:15:26,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=154 2023-09-29T06:15:26,525 INFO [Thread-2713 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 154 completed 2023-09-29T06:15:26,526 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:15:26,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2023-09-29T06:15:26,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=156 2023-09-29T06:15:26,529 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:15:26,529 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:15:26,529 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:15:26,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=156 2023-09-29T06:15:26,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 991924a9bdcd984903381a94960f3885 2023-09-29T06:15:26,650 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 991924a9bdcd984903381a94960f3885 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2023-09-29T06:15:26,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=A 2023-09-29T06:15:26,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:26,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=B 2023-09-29T06:15:26,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:26,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=C 2023-09-29T06:15:26,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:26,655 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/09b81eb9fc3c4ea8905abfc546b60546 is 50, key is test_row_0/A:col10/1695968126650/Put/seqid=0 2023-09-29T06:15:26,659 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/09b81eb9fc3c4ea8905abfc546b60546 2023-09-29T06:15:26,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/86d6eebcbfbe4d069f808ff45b19156f is 50, key is test_row_0/B:col10/1695968126650/Put/seqid=0 2023-09-29T06:15:26,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:26,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968186664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:26,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:26,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968186665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:26,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:26,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968186665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:26,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:26,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968186665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:26,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:26,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968186666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:26,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2023-09-29T06:15:26,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:26,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. as already flushing 2023-09-29T06:15:26,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:26,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(53): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:26,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:26,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:26,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:26,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968186770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:26,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:26,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968186772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:26,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:26,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968186772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:26,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:26,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968186772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:26,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:26,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968186772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:26,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=156 2023-09-29T06:15:26,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2023-09-29T06:15:26,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:26,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. as already flushing 2023-09-29T06:15:26,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:26,835 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(53): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:26,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:26,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:26,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:26,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968186975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:26,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:26,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968186977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:26,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:26,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968186977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:26,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:26,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968186977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:26,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:26,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968186978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:26,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2023-09-29T06:15:26,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:26,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. as already flushing 2023-09-29T06:15:26,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:26,988 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(53): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:26,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:26,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:27,078 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/86d6eebcbfbe4d069f808ff45b19156f 2023-09-29T06:15:27,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/4f6cd91dbe224a7c997d0b9b10b371bb is 50, key is test_row_0/C:col10/1695968126650/Put/seqid=0 2023-09-29T06:15:27,089 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/4f6cd91dbe224a7c997d0b9b10b371bb 2023-09-29T06:15:27,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/09b81eb9fc3c4ea8905abfc546b60546 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/09b81eb9fc3c4ea8905abfc546b60546 2023-09-29T06:15:27,100 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/09b81eb9fc3c4ea8905abfc546b60546, entries=250, sequenceid=405, filesize=16.8 K 2023-09-29T06:15:27,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/86d6eebcbfbe4d069f808ff45b19156f as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/86d6eebcbfbe4d069f808ff45b19156f 2023-09-29T06:15:27,104 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/86d6eebcbfbe4d069f808ff45b19156f, entries=150, sequenceid=405, filesize=12.0 K 2023-09-29T06:15:27,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/4f6cd91dbe224a7c997d0b9b10b371bb as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/4f6cd91dbe224a7c997d0b9b10b371bb 2023-09-29T06:15:27,108 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/4f6cd91dbe224a7c997d0b9b10b371bb, entries=150, sequenceid=405, filesize=12.0 K 2023-09-29T06:15:27,108 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 991924a9bdcd984903381a94960f3885 in 458ms, sequenceid=405, compaction requested=true 2023-09-29T06:15:27,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 991924a9bdcd984903381a94960f3885: 2023-09-29T06:15:27,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 991924a9bdcd984903381a94960f3885:A, priority=-2147483648, current under compaction store size is 1 2023-09-29T06:15:27,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:27,109 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-29T06:15:27,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 991924a9bdcd984903381a94960f3885:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:15:27,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:27,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 991924a9bdcd984903381a94960f3885:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:15:27,109 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-29T06:15:27,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:15:27,110 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54800 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-29T06:15:27,110 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 991924a9bdcd984903381a94960f3885/A is initiating minor compaction (all files) 2023-09-29T06:15:27,110 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 991924a9bdcd984903381a94960f3885/A in TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:27,110 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/1f21f563ba98477399fbab72a2822249, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/6a635bae6dba4a5f82392090d466bd5c, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/ded1b9399eb2459ba6f6edcadc9bdc10, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/09b81eb9fc3c4ea8905abfc546b60546] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp, totalSize=53.5 K 2023-09-29T06:15:27,110 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-29T06:15:27,111 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 991924a9bdcd984903381a94960f3885/B is initiating minor compaction (all files) 2023-09-29T06:15:27,111 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 991924a9bdcd984903381a94960f3885/B in TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:27,111 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/f813ceda39774924875c97a64b342258, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/2ee5669aa9344b6da7efb6c51668eae6, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/553d4f0ddda14276b0f002fc58031ba5, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/86d6eebcbfbe4d069f808ff45b19156f] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp, totalSize=48.8 K 2023-09-29T06:15:27,112 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting f813ceda39774924875c97a64b342258, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1695968121896 2023-09-29T06:15:27,112 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 1f21f563ba98477399fbab72a2822249, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1695968121896 2023-09-29T06:15:27,112 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 2ee5669aa9344b6da7efb6c51668eae6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1695968123018 2023-09-29T06:15:27,112 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 6a635bae6dba4a5f82392090d466bd5c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1695968123018 2023-09-29T06:15:27,112 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 553d4f0ddda14276b0f002fc58031ba5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1695968125177 2023-09-29T06:15:27,112 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting ded1b9399eb2459ba6f6edcadc9bdc10, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1695968125177 2023-09-29T06:15:27,113 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 86d6eebcbfbe4d069f808ff45b19156f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1695968126312 2023-09-29T06:15:27,114 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 09b81eb9fc3c4ea8905abfc546b60546, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1695968126295 2023-09-29T06:15:27,122 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 991924a9bdcd984903381a94960f3885#B#compaction#513 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:27,122 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/9c4e160dda464479bede8cd853cff88a is 50, key is test_row_0/B:col10/1695968126650/Put/seqid=0 2023-09-29T06:15:27,124 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 991924a9bdcd984903381a94960f3885#A#compaction#514 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:27,124 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/b88b1f30e7fe4d4da1136ba075fad4f6 is 50, key is test_row_0/A:col10/1695968126650/Put/seqid=0 2023-09-29T06:15:27,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=156 2023-09-29T06:15:27,135 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/b88b1f30e7fe4d4da1136ba075fad4f6 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/b88b1f30e7fe4d4da1136ba075fad4f6 2023-09-29T06:15:27,139 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 991924a9bdcd984903381a94960f3885/A of 991924a9bdcd984903381a94960f3885 into b88b1f30e7fe4d4da1136ba075fad4f6(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:27,139 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 991924a9bdcd984903381a94960f3885: 2023-09-29T06:15:27,139 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885., storeName=991924a9bdcd984903381a94960f3885/A, priority=12, startTime=1695968127109; duration=0sec 2023-09-29T06:15:27,139 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:15:27,139 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 991924a9bdcd984903381a94960f3885:A 2023-09-29T06:15:27,139 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-29T06:15:27,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2023-09-29T06:15:27,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:27,140 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2819): Flushing 991924a9bdcd984903381a94960f3885 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2023-09-29T06:15:27,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=A 2023-09-29T06:15:27,140 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-29T06:15:27,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:27,140 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 991924a9bdcd984903381a94960f3885/C is initiating minor compaction (all files) 2023-09-29T06:15:27,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=B 2023-09-29T06:15:27,140 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 991924a9bdcd984903381a94960f3885/C in TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:27,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:27,140 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/9cb0431f12f64365b1300340aac54b4c, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/9ca0c6a5f5b74672bd41df8bbd4b89e0, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/a3956bffcc6545db876c7ec705ad7090, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/4f6cd91dbe224a7c997d0b9b10b371bb] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp, totalSize=48.8 K 2023-09-29T06:15:27,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=C 2023-09-29T06:15:27,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:27,141 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 9cb0431f12f64365b1300340aac54b4c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1695968121896 2023-09-29T06:15:27,142 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 9ca0c6a5f5b74672bd41df8bbd4b89e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1695968123018 2023-09-29T06:15:27,142 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting a3956bffcc6545db876c7ec705ad7090, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1695968125177 2023-09-29T06:15:27,142 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 4f6cd91dbe224a7c997d0b9b10b371bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1695968126312 2023-09-29T06:15:27,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/3d3ca6fa7ccb42d9895d516973c550ca is 50, key is test_row_0/A:col10/1695968126663/Put/seqid=0 2023-09-29T06:15:27,152 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/3d3ca6fa7ccb42d9895d516973c550ca 2023-09-29T06:15:27,154 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 991924a9bdcd984903381a94960f3885#C#compaction#516 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:27,154 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/9035cbbc8f784b24bac929329533033b is 50, key is test_row_0/C:col10/1695968126650/Put/seqid=0 2023-09-29T06:15:27,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/6bba29fc403e4e9b8471f2a2203672d9 is 50, key is test_row_0/B:col10/1695968126663/Put/seqid=0 2023-09-29T06:15:27,164 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/9035cbbc8f784b24bac929329533033b as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/9035cbbc8f784b24bac929329533033b 2023-09-29T06:15:27,164 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/6bba29fc403e4e9b8471f2a2203672d9 2023-09-29T06:15:27,169 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 991924a9bdcd984903381a94960f3885/C of 991924a9bdcd984903381a94960f3885 into 9035cbbc8f784b24bac929329533033b(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:27,169 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 991924a9bdcd984903381a94960f3885: 2023-09-29T06:15:27,169 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885., storeName=991924a9bdcd984903381a94960f3885/C, priority=12, startTime=1695968127109; duration=0sec 2023-09-29T06:15:27,169 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:27,169 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 991924a9bdcd984903381a94960f3885:C 2023-09-29T06:15:27,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/8d2c1c1cce154772a896ee1bee9712f7 is 50, key is test_row_0/C:col10/1695968126663/Put/seqid=0 2023-09-29T06:15:27,175 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/8d2c1c1cce154772a896ee1bee9712f7 2023-09-29T06:15:27,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/3d3ca6fa7ccb42d9895d516973c550ca as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/3d3ca6fa7ccb42d9895d516973c550ca 2023-09-29T06:15:27,183 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/3d3ca6fa7ccb42d9895d516973c550ca, entries=150, sequenceid=420, filesize=12.0 K 2023-09-29T06:15:27,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/6bba29fc403e4e9b8471f2a2203672d9 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/6bba29fc403e4e9b8471f2a2203672d9 2023-09-29T06:15:27,187 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/6bba29fc403e4e9b8471f2a2203672d9, entries=150, sequenceid=420, filesize=12.0 K 2023-09-29T06:15:27,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/8d2c1c1cce154772a896ee1bee9712f7 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/8d2c1c1cce154772a896ee1bee9712f7 2023-09-29T06:15:27,193 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/8d2c1c1cce154772a896ee1bee9712f7, entries=150, sequenceid=420, filesize=12.0 K 2023-09-29T06:15:27,193 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3022): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=0 B/0 for 991924a9bdcd984903381a94960f3885 in 53ms, sequenceid=420, compaction requested=false 2023-09-29T06:15:27,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2520): Flush status journal for 991924a9bdcd984903381a94960f3885: 2023-09-29T06:15:27,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:27,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=157 2023-09-29T06:15:27,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=157 2023-09-29T06:15:27,195 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=157, resume processing ppid=156 2023-09-29T06:15:27,195 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=157, ppid=156, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 665 msec 2023-09-29T06:15:27,196 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=156, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 669 msec 2023-09-29T06:15:27,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 991924a9bdcd984903381a94960f3885 2023-09-29T06:15:27,312 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 991924a9bdcd984903381a94960f3885 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2023-09-29T06:15:27,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=A 2023-09-29T06:15:27,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:27,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=B 2023-09-29T06:15:27,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:27,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=C 2023-09-29T06:15:27,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:27,316 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/f5f217b7d9a941a4b5aca0883be796b2 is 50, key is test_row_0/A:col10/1695968127287/Put/seqid=0 2023-09-29T06:15:27,320 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/f5f217b7d9a941a4b5aca0883be796b2 2023-09-29T06:15:27,326 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/02a5141bda964bad8247b00ab1254e68 is 50, key is test_row_0/B:col10/1695968127287/Put/seqid=0 2023-09-29T06:15:27,335 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/02a5141bda964bad8247b00ab1254e68 2023-09-29T06:15:27,341 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/5b3ba24774ac4c4cad6e3ee8628cb886 is 50, key is test_row_0/C:col10/1695968127287/Put/seqid=0 2023-09-29T06:15:27,347 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/5b3ba24774ac4c4cad6e3ee8628cb886 2023-09-29T06:15:27,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/f5f217b7d9a941a4b5aca0883be796b2 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/f5f217b7d9a941a4b5aca0883be796b2 2023-09-29T06:15:27,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:27,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968187346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:27,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:27,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968187348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:27,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:27,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968187348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:27,355 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/f5f217b7d9a941a4b5aca0883be796b2, entries=250, sequenceid=432, filesize=16.8 K 2023-09-29T06:15:27,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/02a5141bda964bad8247b00ab1254e68 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/02a5141bda964bad8247b00ab1254e68 2023-09-29T06:15:27,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:27,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968187351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:27,358 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/02a5141bda964bad8247b00ab1254e68, entries=150, sequenceid=432, filesize=12.0 K 2023-09-29T06:15:27,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/5b3ba24774ac4c4cad6e3ee8628cb886 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/5b3ba24774ac4c4cad6e3ee8628cb886 2023-09-29T06:15:27,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:27,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968187353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:27,362 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/5b3ba24774ac4c4cad6e3ee8628cb886, entries=150, sequenceid=432, filesize=12.0 K 2023-09-29T06:15:27,362 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 991924a9bdcd984903381a94960f3885 in 50ms, sequenceid=432, compaction requested=true 2023-09-29T06:15:27,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 991924a9bdcd984903381a94960f3885: 2023-09-29T06:15:27,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 991924a9bdcd984903381a94960f3885:A, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:15:27,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:27,362 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:15:27,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 991924a9bdcd984903381a94960f3885:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:15:27,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:15:27,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 991924a9bdcd984903381a94960f3885:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:15:27,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2023-09-29T06:15:27,363 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42652 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:15:27,363 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 991924a9bdcd984903381a94960f3885/A is initiating minor compaction (all files) 2023-09-29T06:15:27,363 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 991924a9bdcd984903381a94960f3885/A in TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:27,363 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/b88b1f30e7fe4d4da1136ba075fad4f6, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/3d3ca6fa7ccb42d9895d516973c550ca, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/f5f217b7d9a941a4b5aca0883be796b2] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp, totalSize=41.7 K 2023-09-29T06:15:27,364 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting b88b1f30e7fe4d4da1136ba075fad4f6, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1695968126312 2023-09-29T06:15:27,364 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 3d3ca6fa7ccb42d9895d516973c550ca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1695968126663 2023-09-29T06:15:27,364 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting f5f217b7d9a941a4b5aca0883be796b2, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1695968127285 2023-09-29T06:15:27,369 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 991924a9bdcd984903381a94960f3885#A#compaction#522 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:27,369 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/7ca977e699154686b025eec843b0cee0 is 50, key is test_row_0/A:col10/1695968127287/Put/seqid=0 2023-09-29T06:15:27,384 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/7ca977e699154686b025eec843b0cee0 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/7ca977e699154686b025eec843b0cee0 2023-09-29T06:15:27,388 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 991924a9bdcd984903381a94960f3885/A of 991924a9bdcd984903381a94960f3885 into 7ca977e699154686b025eec843b0cee0(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:27,388 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 991924a9bdcd984903381a94960f3885: 2023-09-29T06:15:27,388 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885., storeName=991924a9bdcd984903381a94960f3885/A, priority=13, startTime=1695968127362; duration=0sec 2023-09-29T06:15:27,388 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2023-09-29T06:15:27,388 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 991924a9bdcd984903381a94960f3885:A 2023-09-29T06:15:27,388 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 4 compacting, 2 eligible, 16 blocking 2023-09-29T06:15:27,389 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2023-09-29T06:15:27,389 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2023-09-29T06:15:27,389 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. because compaction request was cancelled 2023-09-29T06:15:27,389 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 991924a9bdcd984903381a94960f3885:B 2023-09-29T06:15:27,389 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:15:27,390 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37772 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:15:27,390 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 991924a9bdcd984903381a94960f3885/C is initiating minor compaction (all files) 2023-09-29T06:15:27,390 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 991924a9bdcd984903381a94960f3885/C in TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:27,390 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/9035cbbc8f784b24bac929329533033b, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/8d2c1c1cce154772a896ee1bee9712f7, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/5b3ba24774ac4c4cad6e3ee8628cb886] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp, totalSize=36.9 K 2023-09-29T06:15:27,390 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 9035cbbc8f784b24bac929329533033b, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1695968126312 2023-09-29T06:15:27,391 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 8d2c1c1cce154772a896ee1bee9712f7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1695968126663 2023-09-29T06:15:27,391 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 5b3ba24774ac4c4cad6e3ee8628cb886, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1695968127287 2023-09-29T06:15:27,395 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 991924a9bdcd984903381a94960f3885#C#compaction#523 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:27,396 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/f8ebd042db864f91a06754040250f7da is 50, key is test_row_0/C:col10/1695968127287/Put/seqid=0 2023-09-29T06:15:27,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 991924a9bdcd984903381a94960f3885 2023-09-29T06:15:27,456 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 991924a9bdcd984903381a94960f3885 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2023-09-29T06:15:27,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=A 2023-09-29T06:15:27,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:27,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=B 2023-09-29T06:15:27,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:27,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=C 2023-09-29T06:15:27,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:27,461 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/5bd742fd89de48f7837c9f5b60d91266 is 50, key is test_row_0/A:col10/1695968127347/Put/seqid=0 2023-09-29T06:15:27,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:27,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968187463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:27,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:27,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968187463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:27,470 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/5bd742fd89de48f7837c9f5b60d91266 2023-09-29T06:15:27,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:27,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968187464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:27,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:27,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968187465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:27,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:27,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968187466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:27,477 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/1f6e424e2d1f452aaa4057ca8038cbdb is 50, key is test_row_0/B:col10/1695968127347/Put/seqid=0 2023-09-29T06:15:27,484 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/1f6e424e2d1f452aaa4057ca8038cbdb 2023-09-29T06:15:27,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/15ae5f0c83c44df681669e583a780f87 is 50, key is test_row_0/C:col10/1695968127347/Put/seqid=0 2023-09-29T06:15:27,492 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/15ae5f0c83c44df681669e583a780f87 2023-09-29T06:15:27,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/5bd742fd89de48f7837c9f5b60d91266 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/5bd742fd89de48f7837c9f5b60d91266 2023-09-29T06:15:27,499 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/5bd742fd89de48f7837c9f5b60d91266, entries=250, sequenceid=460, filesize=16.8 K 2023-09-29T06:15:27,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/1f6e424e2d1f452aaa4057ca8038cbdb as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/1f6e424e2d1f452aaa4057ca8038cbdb 2023-09-29T06:15:27,503 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/1f6e424e2d1f452aaa4057ca8038cbdb, entries=150, sequenceid=460, filesize=12.0 K 2023-09-29T06:15:27,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/15ae5f0c83c44df681669e583a780f87 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/15ae5f0c83c44df681669e583a780f87 2023-09-29T06:15:27,507 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/15ae5f0c83c44df681669e583a780f87, entries=150, sequenceid=460, filesize=12.0 K 2023-09-29T06:15:27,507 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 991924a9bdcd984903381a94960f3885 in 51ms, sequenceid=460, compaction requested=true 2023-09-29T06:15:27,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 991924a9bdcd984903381a94960f3885: 2023-09-29T06:15:27,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 991924a9bdcd984903381a94960f3885:A, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:15:27,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:15:27,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 991924a9bdcd984903381a94960f3885:B, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:15:27,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2023-09-29T06:15:27,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 991924a9bdcd984903381a94960f3885:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:15:27,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2023-09-29T06:15:27,531 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/9c4e160dda464479bede8cd853cff88a as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/9c4e160dda464479bede8cd853cff88a 2023-09-29T06:15:27,534 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 991924a9bdcd984903381a94960f3885/B of 991924a9bdcd984903381a94960f3885 into 9c4e160dda464479bede8cd853cff88a(size=12.9 K), total size for store is 48.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:27,534 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 991924a9bdcd984903381a94960f3885: 2023-09-29T06:15:27,534 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885., storeName=991924a9bdcd984903381a94960f3885/B, priority=12, startTime=1695968127109; duration=0sec 2023-09-29T06:15:27,534 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2023-09-29T06:15:27,534 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 991924a9bdcd984903381a94960f3885:B 2023-09-29T06:15:27,534 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 991924a9bdcd984903381a94960f3885:B 2023-09-29T06:15:27,535 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 3 compacting, 1 eligible, 16 blocking 2023-09-29T06:15:27,535 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2023-09-29T06:15:27,535 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2023-09-29T06:15:27,535 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. because compaction request was cancelled 2023-09-29T06:15:27,535 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 991924a9bdcd984903381a94960f3885:C 2023-09-29T06:15:27,535 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-29T06:15:27,536 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-29T06:15:27,536 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 991924a9bdcd984903381a94960f3885/B is initiating minor compaction (all files) 2023-09-29T06:15:27,536 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 991924a9bdcd984903381a94960f3885/B in TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:27,536 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/9c4e160dda464479bede8cd853cff88a, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/6bba29fc403e4e9b8471f2a2203672d9, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/02a5141bda964bad8247b00ab1254e68, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/1f6e424e2d1f452aaa4057ca8038cbdb] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp, totalSize=48.9 K 2023-09-29T06:15:27,537 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 9c4e160dda464479bede8cd853cff88a, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1695968126312 2023-09-29T06:15:27,537 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 6bba29fc403e4e9b8471f2a2203672d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1695968126663 2023-09-29T06:15:27,537 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 02a5141bda964bad8247b00ab1254e68, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1695968127287 2023-09-29T06:15:27,538 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 1f6e424e2d1f452aaa4057ca8038cbdb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1695968127347 2023-09-29T06:15:27,545 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 991924a9bdcd984903381a94960f3885#B#compaction#527 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:27,545 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/6e1c9e4d7d904e12b43ac84d15b66f9c is 50, key is test_row_0/B:col10/1695968127347/Put/seqid=0 2023-09-29T06:15:27,553 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/6e1c9e4d7d904e12b43ac84d15b66f9c as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/6e1c9e4d7d904e12b43ac84d15b66f9c 2023-09-29T06:15:27,556 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 991924a9bdcd984903381a94960f3885/B of 991924a9bdcd984903381a94960f3885 into 6e1c9e4d7d904e12b43ac84d15b66f9c(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:27,556 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 991924a9bdcd984903381a94960f3885: 2023-09-29T06:15:27,556 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885., storeName=991924a9bdcd984903381a94960f3885/B, priority=12, startTime=1695968127507; duration=0sec 2023-09-29T06:15:27,556 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:15:27,556 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 991924a9bdcd984903381a94960f3885:B 2023-09-29T06:15:27,556 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2023-09-29T06:15:27,557 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2023-09-29T06:15:27,557 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2023-09-29T06:15:27,557 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. because compaction request was cancelled 2023-09-29T06:15:27,557 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 991924a9bdcd984903381a94960f3885:A 2023-09-29T06:15:27,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 991924a9bdcd984903381a94960f3885 2023-09-29T06:15:27,571 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 991924a9bdcd984903381a94960f3885 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2023-09-29T06:15:27,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=A 2023-09-29T06:15:27,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:27,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=B 2023-09-29T06:15:27,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:27,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=C 2023-09-29T06:15:27,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:27,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/2f2ff08d29e64268a64f47b86849b241 is 50, key is test_row_0/A:col10/1695968127465/Put/seqid=0 2023-09-29T06:15:27,582 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=473 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/2f2ff08d29e64268a64f47b86849b241 2023-09-29T06:15:27,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/65837ac0234b4a3f85438d79cbd705dd is 50, key is test_row_0/B:col10/1695968127465/Put/seqid=0 2023-09-29T06:15:27,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:27,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968187599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:27,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:27,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968187600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:27,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:27,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968187600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:27,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:27,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968187601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:27,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:27,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968187609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:27,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=156 2023-09-29T06:15:27,632 INFO [Thread-2713 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 156 completed 2023-09-29T06:15:27,633 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:15:27,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2023-09-29T06:15:27,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=158 2023-09-29T06:15:27,636 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:15:27,636 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:15:27,636 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:15:27,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:27,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968187710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:27,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:27,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968187710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:27,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:27,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968187710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:27,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:27,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968187710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:27,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:27,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968187717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:27,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=158 2023-09-29T06:15:27,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2023-09-29T06:15:27,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:27,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. as already flushing 2023-09-29T06:15:27,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:27,789 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(53): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:27,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:27,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:27,803 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/f8ebd042db864f91a06754040250f7da as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/f8ebd042db864f91a06754040250f7da 2023-09-29T06:15:27,807 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 991924a9bdcd984903381a94960f3885/C of 991924a9bdcd984903381a94960f3885 into f8ebd042db864f91a06754040250f7da(size=13.0 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:27,807 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 991924a9bdcd984903381a94960f3885: 2023-09-29T06:15:27,807 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885., storeName=991924a9bdcd984903381a94960f3885/C, priority=13, startTime=1695968127363; duration=0sec 2023-09-29T06:15:27,807 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:27,807 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 991924a9bdcd984903381a94960f3885:C 2023-09-29T06:15:27,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:27,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968187913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:27,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:27,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968187913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:27,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:27,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968187914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:27,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:27,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968187915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:27,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:27,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968187919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:27,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=158 2023-09-29T06:15:27,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2023-09-29T06:15:27,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:27,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. as already flushing 2023-09-29T06:15:27,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:27,942 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(53): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:27,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:27,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:27,995 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=473 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/65837ac0234b4a3f85438d79cbd705dd 2023-09-29T06:15:28,001 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/e636092075614d88872368504c5458ce is 50, key is test_row_0/C:col10/1695968127465/Put/seqid=0 2023-09-29T06:15:28,005 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=473 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/e636092075614d88872368504c5458ce 2023-09-29T06:15:28,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/2f2ff08d29e64268a64f47b86849b241 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/2f2ff08d29e64268a64f47b86849b241 2023-09-29T06:15:28,013 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/2f2ff08d29e64268a64f47b86849b241, entries=150, sequenceid=473, filesize=12.0 K 2023-09-29T06:15:28,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/65837ac0234b4a3f85438d79cbd705dd as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/65837ac0234b4a3f85438d79cbd705dd 2023-09-29T06:15:28,018 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/65837ac0234b4a3f85438d79cbd705dd, entries=150, sequenceid=473, filesize=12.0 K 2023-09-29T06:15:28,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/e636092075614d88872368504c5458ce as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/e636092075614d88872368504c5458ce 2023-09-29T06:15:28,023 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/e636092075614d88872368504c5458ce, entries=150, sequenceid=473, filesize=12.0 K 2023-09-29T06:15:28,023 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 991924a9bdcd984903381a94960f3885 in 452ms, sequenceid=473, compaction requested=true 2023-09-29T06:15:28,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 991924a9bdcd984903381a94960f3885: 2023-09-29T06:15:28,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 991924a9bdcd984903381a94960f3885:A, priority=-2147483648, current under compaction store size is 1 2023-09-29T06:15:28,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:28,024 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:15:28,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 991924a9bdcd984903381a94960f3885:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:15:28,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:28,024 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2023-09-29T06:15:28,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 991924a9bdcd984903381a94960f3885:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:15:28,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:15:28,024 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2023-09-29T06:15:28,025 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2023-09-29T06:15:28,025 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. because compaction request was cancelled 2023-09-29T06:15:28,025 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42754 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:15:28,025 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 991924a9bdcd984903381a94960f3885:B 2023-09-29T06:15:28,025 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 991924a9bdcd984903381a94960f3885/A is initiating minor compaction (all files) 2023-09-29T06:15:28,025 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:15:28,025 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 991924a9bdcd984903381a94960f3885/A in TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:28,025 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/7ca977e699154686b025eec843b0cee0, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/5bd742fd89de48f7837c9f5b60d91266, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/2f2ff08d29e64268a64f47b86849b241] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp, totalSize=41.8 K 2023-09-29T06:15:28,025 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 7ca977e699154686b025eec843b0cee0, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1695968127287 2023-09-29T06:15:28,025 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37874 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:15:28,026 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 991924a9bdcd984903381a94960f3885/C is initiating minor compaction (all files) 2023-09-29T06:15:28,026 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 991924a9bdcd984903381a94960f3885/C in TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:28,026 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/f8ebd042db864f91a06754040250f7da, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/15ae5f0c83c44df681669e583a780f87, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/e636092075614d88872368504c5458ce] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp, totalSize=37.0 K 2023-09-29T06:15:28,026 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 5bd742fd89de48f7837c9f5b60d91266, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1695968127337 2023-09-29T06:15:28,026 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting f8ebd042db864f91a06754040250f7da, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1695968127287 2023-09-29T06:15:28,026 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 2f2ff08d29e64268a64f47b86849b241, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1695968127465 2023-09-29T06:15:28,026 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 15ae5f0c83c44df681669e583a780f87, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1695968127347 2023-09-29T06:15:28,027 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting e636092075614d88872368504c5458ce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1695968127465 2023-09-29T06:15:28,032 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 991924a9bdcd984903381a94960f3885#A#compaction#531 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:28,033 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/656a73bc4cb5431797ada246e66a00c9 is 50, key is test_row_0/A:col10/1695968127465/Put/seqid=0 2023-09-29T06:15:28,034 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 991924a9bdcd984903381a94960f3885#C#compaction#532 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:28,034 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/ec70f0898ea741769d618849d6bb33e3 is 50, key is test_row_0/C:col10/1695968127465/Put/seqid=0 2023-09-29T06:15:28,039 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/656a73bc4cb5431797ada246e66a00c9 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/656a73bc4cb5431797ada246e66a00c9 2023-09-29T06:15:28,043 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 991924a9bdcd984903381a94960f3885/A of 991924a9bdcd984903381a94960f3885 into 656a73bc4cb5431797ada246e66a00c9(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:28,043 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 991924a9bdcd984903381a94960f3885: 2023-09-29T06:15:28,043 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885., storeName=991924a9bdcd984903381a94960f3885/A, priority=13, startTime=1695968128024; duration=0sec 2023-09-29T06:15:28,043 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:28,043 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 991924a9bdcd984903381a94960f3885:A 2023-09-29T06:15:28,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2023-09-29T06:15:28,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:28,094 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2819): Flushing 991924a9bdcd984903381a94960f3885 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2023-09-29T06:15:28,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=A 2023-09-29T06:15:28,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:28,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=B 2023-09-29T06:15:28,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:28,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=C 2023-09-29T06:15:28,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:28,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/6ea5880a36a54486a5b8a7ca9fbc1774 is 50, key is test_row_0/A:col10/1695968127599/Put/seqid=0 2023-09-29T06:15:28,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. as already flushing 2023-09-29T06:15:28,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 991924a9bdcd984903381a94960f3885 2023-09-29T06:15:28,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:28,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968188223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:28,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:28,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968188224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:28,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:28,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968188224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:28,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:28,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968188225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:28,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:28,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968188227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:28,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=158 2023-09-29T06:15:28,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:28,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968188329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:28,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:28,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968188329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:28,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:28,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968188329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:28,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:28,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968188332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:28,444 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/ec70f0898ea741769d618849d6bb33e3 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/ec70f0898ea741769d618849d6bb33e3 2023-09-29T06:15:28,449 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 991924a9bdcd984903381a94960f3885/C of 991924a9bdcd984903381a94960f3885 into ec70f0898ea741769d618849d6bb33e3(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:28,449 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 991924a9bdcd984903381a94960f3885: 2023-09-29T06:15:28,449 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885., storeName=991924a9bdcd984903381a94960f3885/C, priority=13, startTime=1695968128024; duration=0sec 2023-09-29T06:15:28,449 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:28,449 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 991924a9bdcd984903381a94960f3885:C 2023-09-29T06:15:28,502 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=500 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/6ea5880a36a54486a5b8a7ca9fbc1774 2023-09-29T06:15:28,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/ac28bdab76a148739d2676541bddc39b is 50, key is test_row_0/B:col10/1695968127599/Put/seqid=0 2023-09-29T06:15:28,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:28,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968188532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:28,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:28,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968188532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:28,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:28,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968188534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:28,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:28,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968188537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:28,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:28,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968188733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:28,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=158 2023-09-29T06:15:28,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:28,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968188840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:28,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:28,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968188840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:28,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:28,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968188840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:28,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:28,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968188841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:28,911 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=500 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/ac28bdab76a148739d2676541bddc39b 2023-09-29T06:15:28,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/cdd332e73674484b8ae7ce788cb73941 is 50, key is test_row_0/C:col10/1695968127599/Put/seqid=0 2023-09-29T06:15:28,923 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=500 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/cdd332e73674484b8ae7ce788cb73941 2023-09-29T06:15:28,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/6ea5880a36a54486a5b8a7ca9fbc1774 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/6ea5880a36a54486a5b8a7ca9fbc1774 2023-09-29T06:15:28,930 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/6ea5880a36a54486a5b8a7ca9fbc1774, entries=150, sequenceid=500, filesize=12.0 K 2023-09-29T06:15:28,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/ac28bdab76a148739d2676541bddc39b as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/ac28bdab76a148739d2676541bddc39b 2023-09-29T06:15:28,935 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/ac28bdab76a148739d2676541bddc39b, entries=150, sequenceid=500, filesize=12.0 K 2023-09-29T06:15:28,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/cdd332e73674484b8ae7ce788cb73941 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/cdd332e73674484b8ae7ce788cb73941 2023-09-29T06:15:28,939 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/cdd332e73674484b8ae7ce788cb73941, entries=150, sequenceid=500, filesize=12.0 K 2023-09-29T06:15:28,939 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3022): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 991924a9bdcd984903381a94960f3885 in 845ms, sequenceid=500, compaction requested=true 2023-09-29T06:15:28,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2520): Flush status journal for 991924a9bdcd984903381a94960f3885: 2023-09-29T06:15:28,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:28,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=159 2023-09-29T06:15:28,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=159 2023-09-29T06:15:28,944 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=159, resume processing ppid=158 2023-09-29T06:15:28,944 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=159, ppid=158, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3070 sec 2023-09-29T06:15:28,945 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=158, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.3110 sec 2023-09-29T06:15:29,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 991924a9bdcd984903381a94960f3885 2023-09-29T06:15:29,346 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 991924a9bdcd984903381a94960f3885 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2023-09-29T06:15:29,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=A 2023-09-29T06:15:29,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:29,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=B 2023-09-29T06:15:29,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:29,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=C 2023-09-29T06:15:29,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:29,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/4bcce8c4214f4285b1ae878de30a20af is 50, key is test_row_0/A:col10/1695968128226/Put/seqid=0 2023-09-29T06:15:29,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:29,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968189380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:29,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:29,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968189382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:29,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:29,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968189383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:29,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:29,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968189387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:29,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:29,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968189488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:29,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:29,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968189488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:29,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:29,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968189488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:29,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:29,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968189492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:29,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:29,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968189691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:29,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:29,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968189691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:29,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:29,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968189692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:29,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:29,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968189697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:29,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=158 2023-09-29T06:15:29,739 INFO [Thread-2713 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 158 completed 2023-09-29T06:15:29,741 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:15:29,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2023-09-29T06:15:29,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=160 2023-09-29T06:15:29,743 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:15:29,744 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:15:29,744 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:15:29,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:29,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42414 deadline: 1695968189746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:29,765 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=513 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/4bcce8c4214f4285b1ae878de30a20af 2023-09-29T06:15:29,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/ce05ee0bd8c745e6b3e106beba9b5714 is 50, key is test_row_0/B:col10/1695968128226/Put/seqid=0 2023-09-29T06:15:29,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=160 2023-09-29T06:15:29,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2023-09-29T06:15:29,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:29,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. as already flushing 2023-09-29T06:15:29,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:29,897 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(53): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:29,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:29,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:29,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:29,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968189995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:29,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:29,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968189995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:29,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:29,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968189997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:30,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:30,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968190001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:30,026 DEBUG [Thread-2714 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x558504ad to 127.0.0.1:49344 2023-09-29T06:15:30,026 DEBUG [Thread-2714 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:15:30,027 DEBUG [Thread-2718 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x04b2228d to 127.0.0.1:49344 2023-09-29T06:15:30,027 DEBUG [Thread-2720 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x5f369608 to 127.0.0.1:49344 2023-09-29T06:15:30,027 DEBUG [Thread-2718 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:15:30,027 DEBUG [Thread-2720 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:15:30,031 DEBUG [Thread-2716 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0c11a915 to 127.0.0.1:49344 2023-09-29T06:15:30,031 DEBUG [Thread-2716 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:15:30,031 DEBUG [Thread-2722 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x3ddf9409 to 127.0.0.1:49344 2023-09-29T06:15:30,031 DEBUG [Thread-2722 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:15:30,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=160 2023-09-29T06:15:30,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2023-09-29T06:15:30,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:30,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. as already flushing 2023-09-29T06:15:30,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:30,049 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(53): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:30,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:30,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:30,175 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=513 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/ce05ee0bd8c745e6b3e106beba9b5714 2023-09-29T06:15:30,180 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/d2ea3c2e2dcd4ecc84c1d334e9282463 is 50, key is test_row_0/C:col10/1695968128226/Put/seqid=0 2023-09-29T06:15:30,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2023-09-29T06:15:30,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:30,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. as already flushing 2023-09-29T06:15:30,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:30,202 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(53): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:30,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:30,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:30,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=160 2023-09-29T06:15:30,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2023-09-29T06:15:30,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:30,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. as already flushing 2023-09-29T06:15:30,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:30,354 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(53): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:30,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:30,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:30,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:30,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:30,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42404 deadline: 1695968190500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:30,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42386 deadline: 1695968190500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:30,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:30,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42480 deadline: 1695968190501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:30,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2023-09-29T06:15:30,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:30,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. as already flushing 2023-09-29T06:15:30,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:30,506 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(53): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:30,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:30,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:30,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:30,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:42426 deadline: 1695968190507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:30,583 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=513 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/d2ea3c2e2dcd4ecc84c1d334e9282463 2023-09-29T06:15:30,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/4bcce8c4214f4285b1ae878de30a20af as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/4bcce8c4214f4285b1ae878de30a20af 2023-09-29T06:15:30,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/4bcce8c4214f4285b1ae878de30a20af, entries=150, sequenceid=513, filesize=12.0 K 2023-09-29T06:15:30,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/ce05ee0bd8c745e6b3e106beba9b5714 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/ce05ee0bd8c745e6b3e106beba9b5714 2023-09-29T06:15:30,592 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/ce05ee0bd8c745e6b3e106beba9b5714, entries=150, sequenceid=513, filesize=12.0 K 2023-09-29T06:15:30,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/d2ea3c2e2dcd4ecc84c1d334e9282463 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/d2ea3c2e2dcd4ecc84c1d334e9282463 2023-09-29T06:15:30,596 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/d2ea3c2e2dcd4ecc84c1d334e9282463, entries=150, sequenceid=513, filesize=12.0 K 2023-09-29T06:15:30,596 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 991924a9bdcd984903381a94960f3885 in 1250ms, sequenceid=513, compaction requested=true 2023-09-29T06:15:30,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 991924a9bdcd984903381a94960f3885: 2023-09-29T06:15:30,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 991924a9bdcd984903381a94960f3885:A, priority=-2147483648, current under compaction store size is 1 2023-09-29T06:15:30,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:30,597 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:15:30,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 991924a9bdcd984903381a94960f3885:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:15:30,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:30,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 991924a9bdcd984903381a94960f3885:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:15:30,597 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-29T06:15:30,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:15:30,597 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37976 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:15:30,597 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 991924a9bdcd984903381a94960f3885/A is initiating minor compaction (all files) 2023-09-29T06:15:30,597 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 991924a9bdcd984903381a94960f3885/A in TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:30,598 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/656a73bc4cb5431797ada246e66a00c9, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/6ea5880a36a54486a5b8a7ca9fbc1774, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/4bcce8c4214f4285b1ae878de30a20af] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp, totalSize=37.1 K 2023-09-29T06:15:30,598 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-29T06:15:30,598 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 991924a9bdcd984903381a94960f3885/B is initiating minor compaction (all files) 2023-09-29T06:15:30,598 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 991924a9bdcd984903381a94960f3885/B in TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:30,598 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 656a73bc4cb5431797ada246e66a00c9, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1695968127465 2023-09-29T06:15:30,598 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/6e1c9e4d7d904e12b43ac84d15b66f9c, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/65837ac0234b4a3f85438d79cbd705dd, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/ac28bdab76a148739d2676541bddc39b, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/ce05ee0bd8c745e6b3e106beba9b5714] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp, totalSize=49.0 K 2023-09-29T06:15:30,598 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 6ea5880a36a54486a5b8a7ca9fbc1774, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=500, earliestPutTs=1695968127583 2023-09-29T06:15:30,598 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 6e1c9e4d7d904e12b43ac84d15b66f9c, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1695968127347 2023-09-29T06:15:30,598 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 4bcce8c4214f4285b1ae878de30a20af, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=513, earliestPutTs=1695968128222 2023-09-29T06:15:30,598 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 65837ac0234b4a3f85438d79cbd705dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1695968127465 2023-09-29T06:15:30,599 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting ac28bdab76a148739d2676541bddc39b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=500, earliestPutTs=1695968127583 2023-09-29T06:15:30,599 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting ce05ee0bd8c745e6b3e106beba9b5714, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=513, earliestPutTs=1695968128222 2023-09-29T06:15:30,603 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 991924a9bdcd984903381a94960f3885#A#compaction#539 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:30,604 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/6942dac448ca47bc9aaf728f0092de27 is 50, key is test_row_0/A:col10/1695968128226/Put/seqid=0 2023-09-29T06:15:30,604 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 991924a9bdcd984903381a94960f3885#B#compaction#540 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:30,605 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/c8d27615a8a64d78b6a93da47cbde1b3 is 50, key is test_row_0/B:col10/1695968128226/Put/seqid=0 2023-09-29T06:15:30,611 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/6942dac448ca47bc9aaf728f0092de27 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/6942dac448ca47bc9aaf728f0092de27 2023-09-29T06:15:30,611 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/c8d27615a8a64d78b6a93da47cbde1b3 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/c8d27615a8a64d78b6a93da47cbde1b3 2023-09-29T06:15:30,614 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 991924a9bdcd984903381a94960f3885/A of 991924a9bdcd984903381a94960f3885 into 6942dac448ca47bc9aaf728f0092de27(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:30,614 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 991924a9bdcd984903381a94960f3885: 2023-09-29T06:15:30,614 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885., storeName=991924a9bdcd984903381a94960f3885/A, priority=13, startTime=1695968130596; duration=0sec 2023-09-29T06:15:30,614 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:15:30,615 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 991924a9bdcd984903381a94960f3885:A 2023-09-29T06:15:30,615 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:15:30,615 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 991924a9bdcd984903381a94960f3885/B of 991924a9bdcd984903381a94960f3885 into c8d27615a8a64d78b6a93da47cbde1b3(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:30,615 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 991924a9bdcd984903381a94960f3885: 2023-09-29T06:15:30,615 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885., storeName=991924a9bdcd984903381a94960f3885/B, priority=12, startTime=1695968130597; duration=0sec 2023-09-29T06:15:30,615 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:30,615 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 991924a9bdcd984903381a94960f3885:B 2023-09-29T06:15:30,615 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37976 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:15:30,615 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 991924a9bdcd984903381a94960f3885/C is initiating minor compaction (all files) 2023-09-29T06:15:30,615 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 991924a9bdcd984903381a94960f3885/C in TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:30,615 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/ec70f0898ea741769d618849d6bb33e3, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/cdd332e73674484b8ae7ce788cb73941, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/d2ea3c2e2dcd4ecc84c1d334e9282463] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp, totalSize=37.1 K 2023-09-29T06:15:30,616 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting ec70f0898ea741769d618849d6bb33e3, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1695968127465 2023-09-29T06:15:30,616 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting cdd332e73674484b8ae7ce788cb73941, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=500, earliestPutTs=1695968127583 2023-09-29T06:15:30,616 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting d2ea3c2e2dcd4ecc84c1d334e9282463, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=513, earliestPutTs=1695968128222 2023-09-29T06:15:30,621 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 991924a9bdcd984903381a94960f3885#C#compaction#541 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:30,621 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/a51ce8af9e7c43baa01c4d6b3773b6d0 is 50, key is test_row_0/C:col10/1695968128226/Put/seqid=0 2023-09-29T06:15:30,626 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/a51ce8af9e7c43baa01c4d6b3773b6d0 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/a51ce8af9e7c43baa01c4d6b3773b6d0 2023-09-29T06:15:30,629 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 991924a9bdcd984903381a94960f3885/C of 991924a9bdcd984903381a94960f3885 into a51ce8af9e7c43baa01c4d6b3773b6d0(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:30,629 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 991924a9bdcd984903381a94960f3885: 2023-09-29T06:15:30,629 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885., storeName=991924a9bdcd984903381a94960f3885/C, priority=13, startTime=1695968130597; duration=0sec 2023-09-29T06:15:30,629 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:30,629 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 991924a9bdcd984903381a94960f3885:C 2023-09-29T06:15:30,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2023-09-29T06:15:30,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:30,658 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2819): Flushing 991924a9bdcd984903381a94960f3885 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2023-09-29T06:15:30,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=A 2023-09-29T06:15:30,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:30,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=B 2023-09-29T06:15:30,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:30,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=C 2023-09-29T06:15:30,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:30,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/c5ed1159d03f4aa89499adb37b706b4b is 50, key is test_row_0/A:col10/1695968129385/Put/seqid=0 2023-09-29T06:15:30,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=160 2023-09-29T06:15:31,064 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=540 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/c5ed1159d03f4aa89499adb37b706b4b 2023-09-29T06:15:31,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/c2bc9f9bec6d437696d55a5dbc4a1528 is 50, key is test_row_0/B:col10/1695968129385/Put/seqid=0 2023-09-29T06:15:31,474 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=540 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/c2bc9f9bec6d437696d55a5dbc4a1528 2023-09-29T06:15:31,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/85bd5edae80448548ac0696ee47f6f90 is 50, key is test_row_0/C:col10/1695968129385/Put/seqid=0 2023-09-29T06:15:31,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 991924a9bdcd984903381a94960f3885 2023-09-29T06:15:31,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. as already flushing 2023-09-29T06:15:31,504 DEBUG [Thread-2711 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x2f0274e0 to 127.0.0.1:49344 2023-09-29T06:15:31,504 DEBUG [Thread-2711 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:15:31,510 DEBUG [Thread-2709 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x1c70815b to 127.0.0.1:49344 2023-09-29T06:15:31,510 DEBUG [Thread-2709 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:15:31,512 DEBUG [Thread-2705 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x67229aff to 127.0.0.1:49344 2023-09-29T06:15:31,512 DEBUG [Thread-2705 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:15:31,514 DEBUG [Thread-2703 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x3a81d834 to 127.0.0.1:49344 2023-09-29T06:15:31,514 DEBUG [Thread-2703 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:15:31,761 DEBUG [Thread-2707 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x02f8d317 to 127.0.0.1:49344 2023-09-29T06:15:31,762 DEBUG [Thread-2707 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:15:31,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=160 2023-09-29T06:15:31,882 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=540 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/85bd5edae80448548ac0696ee47f6f90 2023-09-29T06:15:31,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/c5ed1159d03f4aa89499adb37b706b4b as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/c5ed1159d03f4aa89499adb37b706b4b 2023-09-29T06:15:31,888 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/c5ed1159d03f4aa89499adb37b706b4b, entries=150, sequenceid=540, filesize=12.0 K 2023-09-29T06:15:31,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/c2bc9f9bec6d437696d55a5dbc4a1528 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/c2bc9f9bec6d437696d55a5dbc4a1528 2023-09-29T06:15:31,891 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/c2bc9f9bec6d437696d55a5dbc4a1528, entries=150, sequenceid=540, filesize=12.0 K 2023-09-29T06:15:31,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/85bd5edae80448548ac0696ee47f6f90 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/85bd5edae80448548ac0696ee47f6f90 2023-09-29T06:15:31,894 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/85bd5edae80448548ac0696ee47f6f90, entries=150, sequenceid=540, filesize=12.0 K 2023-09-29T06:15:31,895 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3022): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=33.54 KB/34350 for 991924a9bdcd984903381a94960f3885 in 1237ms, sequenceid=540, compaction requested=false 2023-09-29T06:15:31,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2520): Flush status journal for 991924a9bdcd984903381a94960f3885: 2023-09-29T06:15:31,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:31,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=161 2023-09-29T06:15:31,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=161 2023-09-29T06:15:31,897 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=161, resume processing ppid=160 2023-09-29T06:15:31,897 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=161, ppid=160, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1520 sec 2023-09-29T06:15:31,898 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=160, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 2.1560 sec 2023-09-29T06:15:33,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=160 2023-09-29T06:15:33,848 INFO [Thread-2713 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 160 completed 2023-09-29T06:15:33,848 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2023-09-29T06:15:33,848 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(394): wrote 89 2023-09-29T06:15:33,848 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(394): wrote 84 2023-09-29T06:15:33,848 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(394): wrote 80 2023-09-29T06:15:33,848 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(394): wrote 86 2023-09-29T06:15:33,848 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(394): wrote 92 2023-09-29T06:15:33,848 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(396): Readers: 2023-09-29T06:15:33,848 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2023-09-29T06:15:33,848 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(402): scanned 2948 2023-09-29T06:15:33,848 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(403): verified 8844 rows 2023-09-29T06:15:33,848 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(402): scanned 2924 2023-09-29T06:15:33,848 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(403): verified 8772 rows 2023-09-29T06:15:33,848 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(402): scanned 2937 2023-09-29T06:15:33,849 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(403): verified 8811 rows 2023-09-29T06:15:33,849 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(402): scanned 2941 2023-09-29T06:15:33,849 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(403): verified 8823 rows 2023-09-29T06:15:33,849 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(402): scanned 2944 2023-09-29T06:15:33,849 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(403): verified 8802 rows 2023-09-29T06:15:33,849 INFO [Listener at localhost/42843 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-09-29T06:15:33,849 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x7e27b061 to 127.0.0.1:49344 2023-09-29T06:15:33,849 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:15:33,852 INFO [Listener at localhost/42843 {}] client.HBaseAdmin$18(967): Started disable of TestAcidGuarantees 2023-09-29T06:15:33,853 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster$12(2664): Client=jenkins//172.31.12.81 disable TestAcidGuarantees 2023-09-29T06:15:33,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=162, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=TestAcidGuarantees 2023-09-29T06:15:33,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=162 2023-09-29T06:15:33,857 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968133856"}]},"ts":"1695968133856"} 2023-09-29T06:15:33,857 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2023-09-29T06:15:33,859 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(293): Set TestAcidGuarantees to state=DISABLING 2023-09-29T06:15:33,860 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=991924a9bdcd984903381a94960f3885, UNASSIGN}] 2023-09-29T06:15:33,861 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=163, ppid=162, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=991924a9bdcd984903381a94960f3885, UNASSIGN 2023-09-29T06:15:33,861 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=163 updating hbase:meta row=991924a9bdcd984903381a94960f3885, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:33,862 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=164, ppid=163, state=RUNNABLE; CloseRegionProcedure 991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138}] 2023-09-29T06:15:33,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=162 2023-09-29T06:15:34,014 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] handler.UnassignRegionHandler(119): Close 991924a9bdcd984903381a94960f3885 2023-09-29T06:15:34,014 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.HRegion(1668): Closing 991924a9bdcd984903381a94960f3885, disabling compactions & flushes 2023-09-29T06:15:34,014 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.HRegion(1690): Closing region TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:34,014 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.HRegion(1711): Waiting without time limit for close lock on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:34,014 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.HRegion(1778): Acquired close lock on TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. after waiting 0 ms 2023-09-29T06:15:34,014 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.HRegion(1788): Updates disabled for region TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:34,014 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.HRegion(2819): Flushing 991924a9bdcd984903381a94960f3885 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2023-09-29T06:15:34,014 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=A 2023-09-29T06:15:34,014 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:34,014 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=B 2023-09-29T06:15:34,014 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:34,014 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 991924a9bdcd984903381a94960f3885, store=C 2023-09-29T06:15:34,014 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:34,017 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/f7d21fb6cc2840c3a5dd32c2f3b43942 is 50, key is test_row_0/A:col10/1695968131513/Put/seqid=0 2023-09-29T06:15:34,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=162 2023-09-29T06:15:34,421 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=548 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/f7d21fb6cc2840c3a5dd32c2f3b43942 2023-09-29T06:15:34,426 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/4c3ef14fdb244cbfa5d294277b5f5bf7 is 50, key is test_row_0/B:col10/1695968131513/Put/seqid=0 2023-09-29T06:15:34,429 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=548 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/4c3ef14fdb244cbfa5d294277b5f5bf7 2023-09-29T06:15:34,432 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/8d30aee1f69e44c582ed1a05b31a437e is 50, key is test_row_0/C:col10/1695968131513/Put/seqid=0 2023-09-29T06:15:34,435 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=548 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/8d30aee1f69e44c582ed1a05b31a437e 2023-09-29T06:15:34,438 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/A/f7d21fb6cc2840c3a5dd32c2f3b43942 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/f7d21fb6cc2840c3a5dd32c2f3b43942 2023-09-29T06:15:34,441 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/f7d21fb6cc2840c3a5dd32c2f3b43942, entries=150, sequenceid=548, filesize=12.0 K 2023-09-29T06:15:34,442 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/B/4c3ef14fdb244cbfa5d294277b5f5bf7 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/4c3ef14fdb244cbfa5d294277b5f5bf7 2023-09-29T06:15:34,445 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/4c3ef14fdb244cbfa5d294277b5f5bf7, entries=150, sequenceid=548, filesize=12.0 K 2023-09-29T06:15:34,445 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/.tmp/C/8d30aee1f69e44c582ed1a05b31a437e as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/8d30aee1f69e44c582ed1a05b31a437e 2023-09-29T06:15:34,448 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/8d30aee1f69e44c582ed1a05b31a437e, entries=150, sequenceid=548, filesize=12.0 K 2023-09-29T06:15:34,448 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.HRegion(3022): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 991924a9bdcd984903381a94960f3885 in 434ms, sequenceid=548, compaction requested=true 2023-09-29T06:15:34,450 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/90b9c7b982ef4e069a3a190f97e18d5f, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/4a1adca10e4145b4b7ba9146196abc32, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/dbbf5d562e944afabb3dbe768dd5bc95, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/edea0ed8fbdd4721904a93767dbf0d7b, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/168da897cfd04ef0964d1d5f699e1d86, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/d5f6bcf725074747a723de92351c052c, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/20e9baa79e8149799be4bfdbfb28c5a7, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/64a833b80135463f9446760ac90da910, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/1007f1156b124f05a80a79e0ac53edd8, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/e711c8235f31492eba6f558df55797b5, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/9c35a32d1cc147028be1d562a6929f56, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/c5476dcba8e64f5fa402f88008b64ff3, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/f13b9d58bb604a51ac1df67898c108aa, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/f00c2ed0b0ee45d4bca73ec4eb1e70f6, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/08c0b9df96d54196a01b0e3da179099a, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/549c4d400f114c378d14a2d587f1eb01, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/e3c8ad132ff64e4b93a89c74a9283154, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/117076fcfd1c482f8feedfb9e6ba8318, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/74fc72851b7e4abba5d7ba3e226013e0, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/4c30970feee7484ca1e5b69adaf87fa3, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/3628c3be964a4d55879b376f03063fdd, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/91f6a2a652af4d058b91ac6687ab79e6, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/a1d47f74f1c2422899fa5a3cc919b4d2, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/1f21f563ba98477399fbab72a2822249, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/6a635bae6dba4a5f82392090d466bd5c, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/ded1b9399eb2459ba6f6edcadc9bdc10, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/09b81eb9fc3c4ea8905abfc546b60546, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/b88b1f30e7fe4d4da1136ba075fad4f6, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/3d3ca6fa7ccb42d9895d516973c550ca, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/f5f217b7d9a941a4b5aca0883be796b2, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/7ca977e699154686b025eec843b0cee0, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/5bd742fd89de48f7837c9f5b60d91266, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/656a73bc4cb5431797ada246e66a00c9, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/2f2ff08d29e64268a64f47b86849b241, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/6ea5880a36a54486a5b8a7ca9fbc1774, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/4bcce8c4214f4285b1ae878de30a20af] to archive 2023-09-29T06:15:34,451 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2023-09-29T06:15:34,452 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/90b9c7b982ef4e069a3a190f97e18d5f to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/90b9c7b982ef4e069a3a190f97e18d5f 2023-09-29T06:15:34,453 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/4a1adca10e4145b4b7ba9146196abc32 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/4a1adca10e4145b4b7ba9146196abc32 2023-09-29T06:15:34,454 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/dbbf5d562e944afabb3dbe768dd5bc95 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/dbbf5d562e944afabb3dbe768dd5bc95 2023-09-29T06:15:34,454 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/edea0ed8fbdd4721904a93767dbf0d7b to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/edea0ed8fbdd4721904a93767dbf0d7b 2023-09-29T06:15:34,455 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/168da897cfd04ef0964d1d5f699e1d86 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/168da897cfd04ef0964d1d5f699e1d86 2023-09-29T06:15:34,456 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/d5f6bcf725074747a723de92351c052c to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/d5f6bcf725074747a723de92351c052c 2023-09-29T06:15:34,457 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/20e9baa79e8149799be4bfdbfb28c5a7 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/20e9baa79e8149799be4bfdbfb28c5a7 2023-09-29T06:15:34,458 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/64a833b80135463f9446760ac90da910 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/64a833b80135463f9446760ac90da910 2023-09-29T06:15:34,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=162 2023-09-29T06:15:34,459 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/1007f1156b124f05a80a79e0ac53edd8 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/1007f1156b124f05a80a79e0ac53edd8 2023-09-29T06:15:34,459 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/e711c8235f31492eba6f558df55797b5 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/e711c8235f31492eba6f558df55797b5 2023-09-29T06:15:34,460 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/9c35a32d1cc147028be1d562a6929f56 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/9c35a32d1cc147028be1d562a6929f56 2023-09-29T06:15:34,462 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/c5476dcba8e64f5fa402f88008b64ff3 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/c5476dcba8e64f5fa402f88008b64ff3 2023-09-29T06:15:34,463 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/f13b9d58bb604a51ac1df67898c108aa to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/f13b9d58bb604a51ac1df67898c108aa 2023-09-29T06:15:34,464 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/f00c2ed0b0ee45d4bca73ec4eb1e70f6 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/f00c2ed0b0ee45d4bca73ec4eb1e70f6 2023-09-29T06:15:34,465 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/08c0b9df96d54196a01b0e3da179099a to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/08c0b9df96d54196a01b0e3da179099a 2023-09-29T06:15:34,466 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/549c4d400f114c378d14a2d587f1eb01 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/549c4d400f114c378d14a2d587f1eb01 2023-09-29T06:15:34,467 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/e3c8ad132ff64e4b93a89c74a9283154 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/e3c8ad132ff64e4b93a89c74a9283154 2023-09-29T06:15:34,468 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/117076fcfd1c482f8feedfb9e6ba8318 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/117076fcfd1c482f8feedfb9e6ba8318 2023-09-29T06:15:34,469 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/74fc72851b7e4abba5d7ba3e226013e0 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/74fc72851b7e4abba5d7ba3e226013e0 2023-09-29T06:15:34,470 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/4c30970feee7484ca1e5b69adaf87fa3 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/4c30970feee7484ca1e5b69adaf87fa3 2023-09-29T06:15:34,471 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/3628c3be964a4d55879b376f03063fdd to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/3628c3be964a4d55879b376f03063fdd 2023-09-29T06:15:34,472 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/91f6a2a652af4d058b91ac6687ab79e6 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/91f6a2a652af4d058b91ac6687ab79e6 2023-09-29T06:15:34,473 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/a1d47f74f1c2422899fa5a3cc919b4d2 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/a1d47f74f1c2422899fa5a3cc919b4d2 2023-09-29T06:15:34,474 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/1f21f563ba98477399fbab72a2822249 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/1f21f563ba98477399fbab72a2822249 2023-09-29T06:15:34,475 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/6a635bae6dba4a5f82392090d466bd5c to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/6a635bae6dba4a5f82392090d466bd5c 2023-09-29T06:15:34,476 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/ded1b9399eb2459ba6f6edcadc9bdc10 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/ded1b9399eb2459ba6f6edcadc9bdc10 2023-09-29T06:15:34,477 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/09b81eb9fc3c4ea8905abfc546b60546 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/09b81eb9fc3c4ea8905abfc546b60546 2023-09-29T06:15:34,478 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/b88b1f30e7fe4d4da1136ba075fad4f6 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/b88b1f30e7fe4d4da1136ba075fad4f6 2023-09-29T06:15:34,478 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/3d3ca6fa7ccb42d9895d516973c550ca to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/3d3ca6fa7ccb42d9895d516973c550ca 2023-09-29T06:15:34,479 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/f5f217b7d9a941a4b5aca0883be796b2 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/f5f217b7d9a941a4b5aca0883be796b2 2023-09-29T06:15:34,480 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/7ca977e699154686b025eec843b0cee0 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/7ca977e699154686b025eec843b0cee0 2023-09-29T06:15:34,481 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/5bd742fd89de48f7837c9f5b60d91266 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/5bd742fd89de48f7837c9f5b60d91266 2023-09-29T06:15:34,482 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/656a73bc4cb5431797ada246e66a00c9 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/656a73bc4cb5431797ada246e66a00c9 2023-09-29T06:15:34,483 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/2f2ff08d29e64268a64f47b86849b241 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/2f2ff08d29e64268a64f47b86849b241 2023-09-29T06:15:34,484 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/6ea5880a36a54486a5b8a7ca9fbc1774 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/6ea5880a36a54486a5b8a7ca9fbc1774 2023-09-29T06:15:34,485 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/4bcce8c4214f4285b1ae878de30a20af to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/4bcce8c4214f4285b1ae878de30a20af 2023-09-29T06:15:34,487 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/c0fc6f4c340242b784b3400ef7ec3fb4, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/4cfbf100b19b40a093f6b24a0f5a2ce5, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/369b8ecf18a94122aec03ddd0cc6d85b, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/32b0a6415d7e4e6f8d6cdc3360236926, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/639201b8a8744301bb0506f2ca309136, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/319867d86ceb4dd0911ed15eeeb322fd, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/96c529cd4b1a4af6b40900609c1bc76f, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/f672939870fe479eb53209f479039aba, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/17097b1a029b4864bf7e62c146e394a7, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/f3bca8db01df42039df93f968704b5f8, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/935edb774977402f88922d018f561624, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/959ca23e26c8448aafd0380ca8dc4406, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/90e88b13ba984f5994a5d922c2bd636e, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/3778f02a488545d28b903df262458617, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/bb3f7eebc94f4f7dad37ae74e74de7fc, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/dfeeb62fd6c34a8f877f2fd7808bb410, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/2ff8e04dbb184a8383275e812ea02efd, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/3d9850737c604a89bfba4156c893f683, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/8cc3305265af4a68940b418879baa53c, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/07ec4d1681e64d80b3abf6fe3ba3834c, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/c4bf4767526f459ea6eea1d609dce1e2, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/660288b0de59436dbe0c7f75a8d19927, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/f813ceda39774924875c97a64b342258, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/d4122ffd630c47a9b3c291ec47b6a00b, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/2ee5669aa9344b6da7efb6c51668eae6, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/553d4f0ddda14276b0f002fc58031ba5, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/9c4e160dda464479bede8cd853cff88a, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/86d6eebcbfbe4d069f808ff45b19156f, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/6bba29fc403e4e9b8471f2a2203672d9, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/02a5141bda964bad8247b00ab1254e68, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/6e1c9e4d7d904e12b43ac84d15b66f9c, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/1f6e424e2d1f452aaa4057ca8038cbdb, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/65837ac0234b4a3f85438d79cbd705dd, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/ac28bdab76a148739d2676541bddc39b, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/ce05ee0bd8c745e6b3e106beba9b5714] to archive 2023-09-29T06:15:34,488 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2023-09-29T06:15:34,489 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/c0fc6f4c340242b784b3400ef7ec3fb4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/c0fc6f4c340242b784b3400ef7ec3fb4 2023-09-29T06:15:34,490 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/4cfbf100b19b40a093f6b24a0f5a2ce5 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/4cfbf100b19b40a093f6b24a0f5a2ce5 2023-09-29T06:15:34,491 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/369b8ecf18a94122aec03ddd0cc6d85b to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/369b8ecf18a94122aec03ddd0cc6d85b 2023-09-29T06:15:34,492 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/32b0a6415d7e4e6f8d6cdc3360236926 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/32b0a6415d7e4e6f8d6cdc3360236926 2023-09-29T06:15:34,492 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/639201b8a8744301bb0506f2ca309136 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/639201b8a8744301bb0506f2ca309136 2023-09-29T06:15:34,493 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/319867d86ceb4dd0911ed15eeeb322fd to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/319867d86ceb4dd0911ed15eeeb322fd 2023-09-29T06:15:34,494 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/96c529cd4b1a4af6b40900609c1bc76f to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/96c529cd4b1a4af6b40900609c1bc76f 2023-09-29T06:15:34,495 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/f672939870fe479eb53209f479039aba to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/f672939870fe479eb53209f479039aba 2023-09-29T06:15:34,495 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/17097b1a029b4864bf7e62c146e394a7 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/17097b1a029b4864bf7e62c146e394a7 2023-09-29T06:15:34,496 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/f3bca8db01df42039df93f968704b5f8 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/f3bca8db01df42039df93f968704b5f8 2023-09-29T06:15:34,497 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/935edb774977402f88922d018f561624 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/935edb774977402f88922d018f561624 2023-09-29T06:15:34,498 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/959ca23e26c8448aafd0380ca8dc4406 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/959ca23e26c8448aafd0380ca8dc4406 2023-09-29T06:15:34,499 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/90e88b13ba984f5994a5d922c2bd636e to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/90e88b13ba984f5994a5d922c2bd636e 2023-09-29T06:15:34,500 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/3778f02a488545d28b903df262458617 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/3778f02a488545d28b903df262458617 2023-09-29T06:15:34,500 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/bb3f7eebc94f4f7dad37ae74e74de7fc to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/bb3f7eebc94f4f7dad37ae74e74de7fc 2023-09-29T06:15:34,501 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/dfeeb62fd6c34a8f877f2fd7808bb410 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/dfeeb62fd6c34a8f877f2fd7808bb410 2023-09-29T06:15:34,502 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/2ff8e04dbb184a8383275e812ea02efd to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/2ff8e04dbb184a8383275e812ea02efd 2023-09-29T06:15:34,503 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/3d9850737c604a89bfba4156c893f683 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/3d9850737c604a89bfba4156c893f683 2023-09-29T06:15:34,504 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/8cc3305265af4a68940b418879baa53c to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/8cc3305265af4a68940b418879baa53c 2023-09-29T06:15:34,504 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/07ec4d1681e64d80b3abf6fe3ba3834c to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/07ec4d1681e64d80b3abf6fe3ba3834c 2023-09-29T06:15:34,505 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/c4bf4767526f459ea6eea1d609dce1e2 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/c4bf4767526f459ea6eea1d609dce1e2 2023-09-29T06:15:34,506 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/660288b0de59436dbe0c7f75a8d19927 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/660288b0de59436dbe0c7f75a8d19927 2023-09-29T06:15:34,507 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/f813ceda39774924875c97a64b342258 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/f813ceda39774924875c97a64b342258 2023-09-29T06:15:34,507 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/d4122ffd630c47a9b3c291ec47b6a00b to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/d4122ffd630c47a9b3c291ec47b6a00b 2023-09-29T06:15:34,508 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/2ee5669aa9344b6da7efb6c51668eae6 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/2ee5669aa9344b6da7efb6c51668eae6 2023-09-29T06:15:34,509 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/553d4f0ddda14276b0f002fc58031ba5 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/553d4f0ddda14276b0f002fc58031ba5 2023-09-29T06:15:34,509 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/9c4e160dda464479bede8cd853cff88a to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/9c4e160dda464479bede8cd853cff88a 2023-09-29T06:15:34,510 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/86d6eebcbfbe4d069f808ff45b19156f to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/86d6eebcbfbe4d069f808ff45b19156f 2023-09-29T06:15:34,511 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/6bba29fc403e4e9b8471f2a2203672d9 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/6bba29fc403e4e9b8471f2a2203672d9 2023-09-29T06:15:34,511 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/02a5141bda964bad8247b00ab1254e68 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/02a5141bda964bad8247b00ab1254e68 2023-09-29T06:15:34,512 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/6e1c9e4d7d904e12b43ac84d15b66f9c to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/6e1c9e4d7d904e12b43ac84d15b66f9c 2023-09-29T06:15:34,513 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/1f6e424e2d1f452aaa4057ca8038cbdb to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/1f6e424e2d1f452aaa4057ca8038cbdb 2023-09-29T06:15:34,513 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/65837ac0234b4a3f85438d79cbd705dd to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/65837ac0234b4a3f85438d79cbd705dd 2023-09-29T06:15:34,514 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/ac28bdab76a148739d2676541bddc39b to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/ac28bdab76a148739d2676541bddc39b 2023-09-29T06:15:34,515 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/ce05ee0bd8c745e6b3e106beba9b5714 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/ce05ee0bd8c745e6b3e106beba9b5714 2023-09-29T06:15:34,518 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/2d3917acb9b941beb38bfc679a440e26, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/20a3587796874d6ba5bbdf5ad0e5b3c6, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/6f66805fceeb4fe6a8acbc6a78efb37b, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/23ac67f70863430784c7071046242f7e, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/6248c31548bd440a9ccd9ffd97d41cb8, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/d58f2e727b304843997bb68ba3a9dcb1, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/2c3d35cbce1d4e6a8f683b9a3d6eceb6, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/8d556a8733644627a5c8729e86eff521, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/3cfbb791e1af43da8506e700a119534e, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/8d927e6b7d72406ba1a62e38271543f4, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/bd00c6dc870b418e8bbd9551404a6318, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/e9ef77f047eb40f098a173c68a7f226f, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/344d7841a89d42268b13c8854396e400, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/8eb47ec198f14026957170c698223235, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/5bafcf91d1984f9c8b67d168def1ff3e, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/8737ab3bdb2041bdb5e5ea2f087d3ee9, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/18704b53e32b4f4987b2b9f61cefc12a, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/7609c6376fb944c5801da3acb2dc3b68, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/b1fcb6cdbdb548b68484c29645009b5a, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/24e3ea0bbf1b45c2b1557a800ad4a8db, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/55a6b35eac8d4fabb0128cf0a9a96714, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/f52a47f4740548428be9961164c63f64, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/9cb0431f12f64365b1300340aac54b4c, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/4159035352a645e3a627850e05b5a251, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/9ca0c6a5f5b74672bd41df8bbd4b89e0, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/a3956bffcc6545db876c7ec705ad7090, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/9035cbbc8f784b24bac929329533033b, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/4f6cd91dbe224a7c997d0b9b10b371bb, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/8d2c1c1cce154772a896ee1bee9712f7, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/f8ebd042db864f91a06754040250f7da, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/5b3ba24774ac4c4cad6e3ee8628cb886, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/15ae5f0c83c44df681669e583a780f87, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/ec70f0898ea741769d618849d6bb33e3, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/e636092075614d88872368504c5458ce, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/cdd332e73674484b8ae7ce788cb73941, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/d2ea3c2e2dcd4ecc84c1d334e9282463] to archive 2023-09-29T06:15:34,518 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2023-09-29T06:15:34,520 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/2d3917acb9b941beb38bfc679a440e26 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/2d3917acb9b941beb38bfc679a440e26 2023-09-29T06:15:34,520 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/20a3587796874d6ba5bbdf5ad0e5b3c6 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/20a3587796874d6ba5bbdf5ad0e5b3c6 2023-09-29T06:15:34,521 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/6f66805fceeb4fe6a8acbc6a78efb37b to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/6f66805fceeb4fe6a8acbc6a78efb37b 2023-09-29T06:15:34,522 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/23ac67f70863430784c7071046242f7e to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/23ac67f70863430784c7071046242f7e 2023-09-29T06:15:34,523 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/6248c31548bd440a9ccd9ffd97d41cb8 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/6248c31548bd440a9ccd9ffd97d41cb8 2023-09-29T06:15:34,523 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/d58f2e727b304843997bb68ba3a9dcb1 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/d58f2e727b304843997bb68ba3a9dcb1 2023-09-29T06:15:34,524 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/2c3d35cbce1d4e6a8f683b9a3d6eceb6 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/2c3d35cbce1d4e6a8f683b9a3d6eceb6 2023-09-29T06:15:34,525 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/8d556a8733644627a5c8729e86eff521 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/8d556a8733644627a5c8729e86eff521 2023-09-29T06:15:34,526 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/3cfbb791e1af43da8506e700a119534e to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/3cfbb791e1af43da8506e700a119534e 2023-09-29T06:15:34,527 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/8d927e6b7d72406ba1a62e38271543f4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/8d927e6b7d72406ba1a62e38271543f4 2023-09-29T06:15:34,527 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/bd00c6dc870b418e8bbd9551404a6318 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/bd00c6dc870b418e8bbd9551404a6318 2023-09-29T06:15:34,528 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/e9ef77f047eb40f098a173c68a7f226f to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/e9ef77f047eb40f098a173c68a7f226f 2023-09-29T06:15:34,529 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/344d7841a89d42268b13c8854396e400 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/344d7841a89d42268b13c8854396e400 2023-09-29T06:15:34,530 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/8eb47ec198f14026957170c698223235 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/8eb47ec198f14026957170c698223235 2023-09-29T06:15:34,531 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/5bafcf91d1984f9c8b67d168def1ff3e to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/5bafcf91d1984f9c8b67d168def1ff3e 2023-09-29T06:15:34,531 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/8737ab3bdb2041bdb5e5ea2f087d3ee9 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/8737ab3bdb2041bdb5e5ea2f087d3ee9 2023-09-29T06:15:34,532 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/18704b53e32b4f4987b2b9f61cefc12a to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/18704b53e32b4f4987b2b9f61cefc12a 2023-09-29T06:15:34,533 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/7609c6376fb944c5801da3acb2dc3b68 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/7609c6376fb944c5801da3acb2dc3b68 2023-09-29T06:15:34,534 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/b1fcb6cdbdb548b68484c29645009b5a to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/b1fcb6cdbdb548b68484c29645009b5a 2023-09-29T06:15:34,534 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/24e3ea0bbf1b45c2b1557a800ad4a8db to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/24e3ea0bbf1b45c2b1557a800ad4a8db 2023-09-29T06:15:34,535 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/55a6b35eac8d4fabb0128cf0a9a96714 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/55a6b35eac8d4fabb0128cf0a9a96714 2023-09-29T06:15:34,536 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/f52a47f4740548428be9961164c63f64 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/f52a47f4740548428be9961164c63f64 2023-09-29T06:15:34,537 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/9cb0431f12f64365b1300340aac54b4c to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/9cb0431f12f64365b1300340aac54b4c 2023-09-29T06:15:34,537 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/4159035352a645e3a627850e05b5a251 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/4159035352a645e3a627850e05b5a251 2023-09-29T06:15:34,538 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/9ca0c6a5f5b74672bd41df8bbd4b89e0 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/9ca0c6a5f5b74672bd41df8bbd4b89e0 2023-09-29T06:15:34,539 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/a3956bffcc6545db876c7ec705ad7090 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/a3956bffcc6545db876c7ec705ad7090 2023-09-29T06:15:34,540 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/9035cbbc8f784b24bac929329533033b to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/9035cbbc8f784b24bac929329533033b 2023-09-29T06:15:34,540 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/4f6cd91dbe224a7c997d0b9b10b371bb to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/4f6cd91dbe224a7c997d0b9b10b371bb 2023-09-29T06:15:34,541 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/8d2c1c1cce154772a896ee1bee9712f7 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/8d2c1c1cce154772a896ee1bee9712f7 2023-09-29T06:15:34,542 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/f8ebd042db864f91a06754040250f7da to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/f8ebd042db864f91a06754040250f7da 2023-09-29T06:15:34,543 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/5b3ba24774ac4c4cad6e3ee8628cb886 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/5b3ba24774ac4c4cad6e3ee8628cb886 2023-09-29T06:15:34,544 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/15ae5f0c83c44df681669e583a780f87 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/15ae5f0c83c44df681669e583a780f87 2023-09-29T06:15:34,544 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/ec70f0898ea741769d618849d6bb33e3 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/ec70f0898ea741769d618849d6bb33e3 2023-09-29T06:15:34,545 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/e636092075614d88872368504c5458ce to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/e636092075614d88872368504c5458ce 2023-09-29T06:15:34,546 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/cdd332e73674484b8ae7ce788cb73941 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/cdd332e73674484b8ae7ce788cb73941 2023-09-29T06:15:34,547 DEBUG [StoreCloser-TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/d2ea3c2e2dcd4ecc84c1d334e9282463 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/d2ea3c2e2dcd4ecc84c1d334e9282463 2023-09-29T06:15:34,550 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/recovered.edits/551.seqid, newMaxSeqId=551, maxSeqId=1 2023-09-29T06:15:34,551 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.HRegion(1909): Closed TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885. 2023-09-29T06:15:34,551 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.HRegion(1622): Region close journal for 991924a9bdcd984903381a94960f3885: 2023-09-29T06:15:34,552 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] handler.UnassignRegionHandler(163): Closed 991924a9bdcd984903381a94960f3885 2023-09-29T06:15:34,553 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=163 updating hbase:meta row=991924a9bdcd984903381a94960f3885, regionState=CLOSED 2023-09-29T06:15:34,554 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=164, resume processing ppid=163 2023-09-29T06:15:34,554 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=164, ppid=163, state=SUCCESS; CloseRegionProcedure 991924a9bdcd984903381a94960f3885, server=jenkins-hbase3.apache.org,39783,1695967978138 in 691 msec 2023-09-29T06:15:34,555 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=163, resume processing ppid=162 2023-09-29T06:15:34,555 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=163, ppid=162, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=991924a9bdcd984903381a94960f3885, UNASSIGN in 694 msec 2023-09-29T06:15:34,556 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968134556"}]},"ts":"1695968134556"} 2023-09-29T06:15:34,556 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2023-09-29T06:15:34,558 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(305): Set TestAcidGuarantees to state=DISABLED 2023-09-29T06:15:34,559 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=162, state=SUCCESS; DisableTableProcedure table=TestAcidGuarantees in 706 msec 2023-09-29T06:15:34,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=162 2023-09-29T06:15:34,959 INFO [Listener at localhost/42843 {}] client.HBaseAdmin$TableFuture(3677): Operation: DISABLE, Table Name: default:TestAcidGuarantees, procId: 162 completed 2023-09-29T06:15:34,960 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster$5(2448): Client=jenkins//172.31.12.81 delete TestAcidGuarantees 2023-09-29T06:15:34,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=165, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=TestAcidGuarantees 2023-09-29T06:15:34,962 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=165, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2023-09-29T06:15:34,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=165 2023-09-29T06:15:34,962 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=165, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2023-09-29T06:15:34,964 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885 2023-09-29T06:15:34,966 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A, FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B, FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C, FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/recovered.edits] 2023-09-29T06:15:34,969 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/6942dac448ca47bc9aaf728f0092de27 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/6942dac448ca47bc9aaf728f0092de27 2023-09-29T06:15:34,970 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/c5ed1159d03f4aa89499adb37b706b4b to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/c5ed1159d03f4aa89499adb37b706b4b 2023-09-29T06:15:34,971 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/f7d21fb6cc2840c3a5dd32c2f3b43942 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/A/f7d21fb6cc2840c3a5dd32c2f3b43942 2023-09-29T06:15:34,973 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/4c3ef14fdb244cbfa5d294277b5f5bf7 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/4c3ef14fdb244cbfa5d294277b5f5bf7 2023-09-29T06:15:34,974 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/c2bc9f9bec6d437696d55a5dbc4a1528 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/c2bc9f9bec6d437696d55a5dbc4a1528 2023-09-29T06:15:34,975 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/c8d27615a8a64d78b6a93da47cbde1b3 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/B/c8d27615a8a64d78b6a93da47cbde1b3 2023-09-29T06:15:34,976 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/85bd5edae80448548ac0696ee47f6f90 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/85bd5edae80448548ac0696ee47f6f90 2023-09-29T06:15:34,977 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/8d30aee1f69e44c582ed1a05b31a437e to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/8d30aee1f69e44c582ed1a05b31a437e 2023-09-29T06:15:34,978 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/a51ce8af9e7c43baa01c4d6b3773b6d0 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/C/a51ce8af9e7c43baa01c4d6b3773b6d0 2023-09-29T06:15:34,980 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/recovered.edits/551.seqid to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885/recovered.edits/551.seqid 2023-09-29T06:15:34,981 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/991924a9bdcd984903381a94960f3885 2023-09-29T06:15:34,981 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(293): Archived TestAcidGuarantees regions 2023-09-29T06:15:34,983 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=165, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2023-09-29T06:15:34,984 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2023-09-29T06:15:34,985 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(387): Removing 'TestAcidGuarantees' descriptor. 2023-09-29T06:15:34,986 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=165, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2023-09-29T06:15:34,986 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(377): Removing 'TestAcidGuarantees' from region states. 2023-09-29T06:15:34,986 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1695968134986"}]},"ts":"9223372036854775807"} 2023-09-29T06:15:34,987 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-09-29T06:15:34,987 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 991924a9bdcd984903381a94960f3885, NAME => 'TestAcidGuarantees,,1695968107849.991924a9bdcd984903381a94960f3885.', STARTKEY => '', ENDKEY => ''}] 2023-09-29T06:15:34,987 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(381): Marking 'TestAcidGuarantees' as deleted. 2023-09-29T06:15:34,987 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1695968134987"}]},"ts":"9223372036854775807"} 2023-09-29T06:15:34,989 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1678): Deleted table TestAcidGuarantees state from META 2023-09-29T06:15:34,992 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(127): Finished pid=165, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2023-09-29T06:15:34,992 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=165, state=SUCCESS; DeleteTableProcedure table=TestAcidGuarantees in 32 msec 2023-09-29T06:15:35,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=165 2023-09-29T06:15:35,063 INFO [Listener at localhost/42843 {}] client.HBaseAdmin$TableFuture(3677): Operation: DELETE, Table Name: default:TestAcidGuarantees, procId: 165 completed 2023-09-29T06:15:35,075 INFO [Listener at localhost/42843 {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithAdaptivePolicy#testScanAtomicity Thread=221 (was 225), OpenFileDescriptor=600 (was 574) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=505 (was 608), ProcessCount=167 (was 167), AvailableMemoryMB=1395 (was 1418) 2023-09-29T06:15:35,086 INFO [Listener at localhost/42843 {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithAdaptivePolicy#testMobGetAtomicity Thread=221, OpenFileDescriptor=600, MaxFileDescriptor=60000, SystemLoadAverage=505, ProcessCount=167, AvailableMemoryMB=1395 2023-09-29T06:15:35,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] util.TableDescriptorChecker(342): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2023-09-29T06:15:35,088 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'ADAPTIVE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-29T06:15:35,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=166, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestAcidGuarantees 2023-09-29T06:15:35,089 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=166, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2023-09-29T06:15:35,089 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:15:35,090 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 166 2023-09-29T06:15:35,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=166 2023-09-29T06:15:35,090 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=166, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-29T06:15:35,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=166 2023-09-29T06:15:35,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=166 2023-09-29T06:15:35,496 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'ADAPTIVE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a 2023-09-29T06:15:35,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=166 2023-09-29T06:15:35,901 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(888): Instantiated TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:15:35,901 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1668): Closing 0f1c71e3f887ce47e67c17dad51fd610, disabling compactions & flushes 2023-09-29T06:15:35,901 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1690): Closing region TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:35,901 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:35,901 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. after waiting 0 ms 2023-09-29T06:15:35,901 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:35,901 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1909): Closed TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:35,901 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1622): Region close journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:35,902 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=166, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2023-09-29T06:15:35,902 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1695968135902"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1695968135902"}]},"ts":"1695968135902"} 2023-09-29T06:15:35,903 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-29T06:15:35,903 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=166, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-29T06:15:35,904 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968135903"}]},"ts":"1695968135903"} 2023-09-29T06:15:35,904 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2023-09-29T06:15:35,908 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=0f1c71e3f887ce47e67c17dad51fd610, ASSIGN}] 2023-09-29T06:15:35,910 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=167, ppid=166, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=0f1c71e3f887ce47e67c17dad51fd610, ASSIGN 2023-09-29T06:15:35,910 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=167, ppid=166, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0f1c71e3f887ce47e67c17dad51fd610, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,39783,1695967978138; forceNewPlan=false, retain=false 2023-09-29T06:15:36,062 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=167 updating hbase:meta row=0f1c71e3f887ce47e67c17dad51fd610, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:36,063 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=168, ppid=167, state=RUNNABLE; OpenRegionProcedure 0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138}] 2023-09-29T06:15:36,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=166 2023-09-29T06:15:36,217 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=168}] handler.AssignRegionHandler(130): Open TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:36,217 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=168}] regionserver.HRegion(7254): Opening region: {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} 2023-09-29T06:15:36,217 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=168}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:36,217 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=168}] regionserver.HRegion(888): Instantiated TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:15:36,217 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=168}] regionserver.HRegion(7296): checking encryption for 0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:36,217 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=168}] regionserver.HRegion(7299): checking classloading for 0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:36,218 INFO [StoreOpener-0f1c71e3f887ce47e67c17dad51fd610-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:36,219 INFO [StoreOpener-0f1c71e3f887ce47e67c17dad51fd610-1 {}] regionserver.CompactingMemStore(121): Store=A, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2023-09-29T06:15:36,219 INFO [StoreOpener-0f1c71e3f887ce47e67c17dad51fd610-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0f1c71e3f887ce47e67c17dad51fd610 columnFamilyName A 2023-09-29T06:15:36,219 DEBUG [StoreOpener-0f1c71e3f887ce47e67c17dad51fd610-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:15:36,220 INFO [StoreOpener-0f1c71e3f887ce47e67c17dad51fd610-1 {}] regionserver.HStore(324): Store=0f1c71e3f887ce47e67c17dad51fd610/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:15:36,220 INFO [StoreOpener-0f1c71e3f887ce47e67c17dad51fd610-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:36,221 INFO [StoreOpener-0f1c71e3f887ce47e67c17dad51fd610-1 {}] regionserver.CompactingMemStore(121): Store=B, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2023-09-29T06:15:36,221 INFO [StoreOpener-0f1c71e3f887ce47e67c17dad51fd610-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0f1c71e3f887ce47e67c17dad51fd610 columnFamilyName B 2023-09-29T06:15:36,221 DEBUG [StoreOpener-0f1c71e3f887ce47e67c17dad51fd610-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:15:36,221 INFO [StoreOpener-0f1c71e3f887ce47e67c17dad51fd610-1 {}] regionserver.HStore(324): Store=0f1c71e3f887ce47e67c17dad51fd610/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:15:36,221 INFO [StoreOpener-0f1c71e3f887ce47e67c17dad51fd610-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:36,222 INFO [StoreOpener-0f1c71e3f887ce47e67c17dad51fd610-1 {}] regionserver.CompactingMemStore(121): Store=C, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2023-09-29T06:15:36,222 INFO [StoreOpener-0f1c71e3f887ce47e67c17dad51fd610-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0f1c71e3f887ce47e67c17dad51fd610 columnFamilyName C 2023-09-29T06:15:36,222 DEBUG [StoreOpener-0f1c71e3f887ce47e67c17dad51fd610-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:15:36,223 INFO [StoreOpener-0f1c71e3f887ce47e67c17dad51fd610-1 {}] regionserver.HStore(324): Store=0f1c71e3f887ce47e67c17dad51fd610/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:15:36,223 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=168}] regionserver.HRegion(1169): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:36,223 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=168}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:36,224 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=168}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:36,225 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=168}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2023-09-29T06:15:36,225 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=168}] regionserver.HRegion(1076): writing seq id for 0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:36,226 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=168}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-29T06:15:36,227 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=168}] regionserver.HRegion(1093): Opened 0f1c71e3f887ce47e67c17dad51fd610; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62807714, jitterRate=-0.06409212946891785}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2023-09-29T06:15:36,227 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=168}] regionserver.HRegion(992): Region open journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:36,228 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=168}] regionserver.HRegionServer(2556): Post open deploy tasks for TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610., pid=168, masterSystemTime=1695968136214 2023-09-29T06:15:36,229 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=168}] regionserver.HRegionServer(2583): Finished post open deploy task for TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:36,229 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=168}] handler.AssignRegionHandler(158): Opened TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:36,229 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=167 updating hbase:meta row=0f1c71e3f887ce47e67c17dad51fd610, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:36,231 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=168, resume processing ppid=167 2023-09-29T06:15:36,231 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=168, ppid=167, state=SUCCESS; OpenRegionProcedure 0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 in 167 msec 2023-09-29T06:15:36,232 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=167, resume processing ppid=166 2023-09-29T06:15:36,232 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=167, ppid=166, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=0f1c71e3f887ce47e67c17dad51fd610, ASSIGN in 323 msec 2023-09-29T06:15:36,232 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=166, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-29T06:15:36,233 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968136232"}]},"ts":"1695968136232"} 2023-09-29T06:15:36,233 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2023-09-29T06:15:36,235 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=166, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2023-09-29T06:15:36,236 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=166, state=SUCCESS; CreateTableProcedure table=TestAcidGuarantees in 1.1470 sec 2023-09-29T06:15:37,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=166 2023-09-29T06:15:37,194 INFO [Listener at localhost/42843 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: default:TestAcidGuarantees, procId: 166 completed 2023-09-29T06:15:37,195 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x522ff045 to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:15:37,201 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cdf5622, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:15:37,202 DEBUG [Listener at localhost/42843 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:15:37,204 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:37024, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:15:37,205 DEBUG [Listener at localhost/42843 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-29T06:15:37,206 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:34236, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-29T06:15:37,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] util.TableDescriptorChecker(342): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2023-09-29T06:15:37,207 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster$13(2700): Client=jenkins//172.31.12.81 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'ADAPTIVE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'ADAPTIVE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-29T06:15:37,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=169, state=RUNNABLE:MODIFY_TABLE_PREPARE; ModifyTableProcedure table=TestAcidGuarantees 2023-09-29T06:15:37,220 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.963 2023-09-29T06:15:37,221 INFO [PEWorker-4 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.999 2023-09-29T06:15:37,222 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=170, ppid=169, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2023-09-29T06:15:37,224 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=0f1c71e3f887ce47e67c17dad51fd610, REOPEN/MOVE}] 2023-09-29T06:15:37,224 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=171, ppid=170, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=0f1c71e3f887ce47e67c17dad51fd610, REOPEN/MOVE 2023-09-29T06:15:37,225 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=171 updating hbase:meta row=0f1c71e3f887ce47e67c17dad51fd610, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:37,226 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=172, ppid=171, state=RUNNABLE; CloseRegionProcedure 0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138}] 2023-09-29T06:15:37,377 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=172}] handler.UnassignRegionHandler(119): Close 0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:37,378 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=172}] regionserver.HRegion(1668): Closing 0f1c71e3f887ce47e67c17dad51fd610, disabling compactions & flushes 2023-09-29T06:15:37,378 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=172}] regionserver.HRegion(1690): Closing region TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:37,378 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=172}] regionserver.HRegion(1711): Waiting without time limit for close lock on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:37,378 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=172}] regionserver.HRegion(1778): Acquired close lock on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. after waiting 0 ms 2023-09-29T06:15:37,378 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=172}] regionserver.HRegion(1788): Updates disabled for region TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:37,381 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=172}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-09-29T06:15:37,381 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=172}] regionserver.HRegion(1909): Closed TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:37,381 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=172}] regionserver.HRegion(1622): Region close journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:37,381 WARN [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=172}] regionserver.HRegionServer(3735): Not adding moved region record: 0f1c71e3f887ce47e67c17dad51fd610 to self. 2023-09-29T06:15:37,382 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=172}] handler.UnassignRegionHandler(163): Closed 0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:37,383 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=171 updating hbase:meta row=0f1c71e3f887ce47e67c17dad51fd610, regionState=CLOSED 2023-09-29T06:15:37,385 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=172, resume processing ppid=171 2023-09-29T06:15:37,385 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=172, ppid=171, state=SUCCESS; CloseRegionProcedure 0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 in 157 msec 2023-09-29T06:15:37,385 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=171, ppid=170, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0f1c71e3f887ce47e67c17dad51fd610, REOPEN/MOVE; state=CLOSED, location=jenkins-hbase3.apache.org,39783,1695967978138; forceNewPlan=false, retain=true 2023-09-29T06:15:37,460 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2023-09-29T06:15:37,535 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=171 updating hbase:meta row=0f1c71e3f887ce47e67c17dad51fd610, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:37,536 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=173, ppid=171, state=RUNNABLE; OpenRegionProcedure 0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138}] 2023-09-29T06:15:37,690 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=173}] handler.AssignRegionHandler(130): Open TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:37,691 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=173}] regionserver.HRegion(7254): Opening region: {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} 2023-09-29T06:15:37,691 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=173}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:37,691 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=173}] regionserver.HRegion(888): Instantiated TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-29T06:15:37,691 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=173}] regionserver.HRegion(7296): checking encryption for 0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:37,691 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=173}] regionserver.HRegion(7299): checking classloading for 0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:37,692 INFO [StoreOpener-0f1c71e3f887ce47e67c17dad51fd610-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:37,693 INFO [StoreOpener-0f1c71e3f887ce47e67c17dad51fd610-1 {}] regionserver.CompactingMemStore(121): Store=A, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2023-09-29T06:15:37,693 INFO [StoreOpener-0f1c71e3f887ce47e67c17dad51fd610-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0f1c71e3f887ce47e67c17dad51fd610 columnFamilyName A 2023-09-29T06:15:37,694 DEBUG [StoreOpener-0f1c71e3f887ce47e67c17dad51fd610-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:15:37,694 INFO [StoreOpener-0f1c71e3f887ce47e67c17dad51fd610-1 {}] regionserver.HStore(324): Store=0f1c71e3f887ce47e67c17dad51fd610/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:15:37,694 INFO [StoreOpener-0f1c71e3f887ce47e67c17dad51fd610-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:37,695 INFO [StoreOpener-0f1c71e3f887ce47e67c17dad51fd610-1 {}] regionserver.CompactingMemStore(121): Store=B, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2023-09-29T06:15:37,695 INFO [StoreOpener-0f1c71e3f887ce47e67c17dad51fd610-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0f1c71e3f887ce47e67c17dad51fd610 columnFamilyName B 2023-09-29T06:15:37,695 DEBUG [StoreOpener-0f1c71e3f887ce47e67c17dad51fd610-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:15:37,695 INFO [StoreOpener-0f1c71e3f887ce47e67c17dad51fd610-1 {}] regionserver.HStore(324): Store=0f1c71e3f887ce47e67c17dad51fd610/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:15:37,695 INFO [StoreOpener-0f1c71e3f887ce47e67c17dad51fd610-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:37,696 INFO [StoreOpener-0f1c71e3f887ce47e67c17dad51fd610-1 {}] regionserver.CompactingMemStore(121): Store=C, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=ADAPTIVE, pipelineThreshold=2, compactionCellMax=10 2023-09-29T06:15:37,696 INFO [StoreOpener-0f1c71e3f887ce47e67c17dad51fd610-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0f1c71e3f887ce47e67c17dad51fd610 columnFamilyName C 2023-09-29T06:15:37,696 DEBUG [StoreOpener-0f1c71e3f887ce47e67c17dad51fd610-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-29T06:15:37,696 INFO [StoreOpener-0f1c71e3f887ce47e67c17dad51fd610-1 {}] regionserver.HStore(324): Store=0f1c71e3f887ce47e67c17dad51fd610/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-29T06:15:37,697 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=173}] regionserver.HRegion(1169): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:37,697 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=173}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:37,698 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=173}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:37,699 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=173}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2023-09-29T06:15:37,700 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=173}] regionserver.HRegion(1076): writing seq id for 0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:37,700 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=173}] regionserver.HRegion(1093): Opened 0f1c71e3f887ce47e67c17dad51fd610; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59000283, jitterRate=-0.12082727253437042}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2023-09-29T06:15:37,701 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=173}] regionserver.HRegion(992): Region open journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:37,701 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=173}] regionserver.HRegionServer(2556): Post open deploy tasks for TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610., pid=173, masterSystemTime=1695968137687 2023-09-29T06:15:37,703 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=173}] regionserver.HRegionServer(2583): Finished post open deploy task for TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:37,703 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=173}] handler.AssignRegionHandler(158): Opened TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:37,703 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=171 updating hbase:meta row=0f1c71e3f887ce47e67c17dad51fd610, regionState=OPEN, openSeqNum=5, regionLocation=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:37,705 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=173, resume processing ppid=171 2023-09-29T06:15:37,705 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=173, ppid=171, state=SUCCESS; OpenRegionProcedure 0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 in 168 msec 2023-09-29T06:15:37,706 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=171, resume processing ppid=170 2023-09-29T06:15:37,706 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=171, ppid=170, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=0f1c71e3f887ce47e67c17dad51fd610, REOPEN/MOVE in 481 msec 2023-09-29T06:15:37,707 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=170, resume processing ppid=169 2023-09-29T06:15:37,707 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=170, ppid=169, state=SUCCESS; ReopenTableRegionsProcedure table=TestAcidGuarantees in 484 msec 2023-09-29T06:15:37,708 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=169, state=SUCCESS; ModifyTableProcedure table=TestAcidGuarantees in 501 msec 2023-09-29T06:15:37,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=169 2023-09-29T06:15:37,710 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x51685a58 to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:15:37,715 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ce835e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:15:37,715 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x2aaa0778 to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:15:37,719 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@398322ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:15:37,720 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x5e13bd56 to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:15:37,723 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24f8e02b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:15:37,724 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x48436ecd to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:15:37,729 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b79939e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:15:37,729 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0d1a704d to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:15:37,733 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e7c5225, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:15:37,734 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x3116ea32 to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:15:37,740 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dc76b4f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:15:37,741 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0e0e44d4 to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:15:37,745 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@84f685d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:15:37,746 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x29a1b140 to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:15:37,752 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55756844, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:15:37,753 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x3f4e9059 to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:15:37,756 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bc11d41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:15:37,757 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x1c03d8f4 to 127.0.0.1:49344 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-29T06:15:37,761 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57044f75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-09-29T06:15:37,767 DEBUG [hconnection-0x6fedfac-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:15:37,768 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:15:37,768 DEBUG [hconnection-0x32e0abdf-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:15:37,768 DEBUG [hconnection-0x13b278b2-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:15:37,768 DEBUG [hconnection-0x74d6a146-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:15:37,768 DEBUG [hconnection-0x70b48fab-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:15:37,774 DEBUG [hconnection-0x437544c9-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:15:37,774 DEBUG [hconnection-0x31884975-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:15:37,774 DEBUG [hconnection-0x66860104-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:15:37,774 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:37026, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:15:37,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=174, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees 2023-09-29T06:15:37,775 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:37032, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:15:37,775 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:37046, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:15:37,776 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:37058, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:15:37,776 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:37034, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:15:37,776 DEBUG [hconnection-0x626ed18-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:15:37,778 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:37068, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:15:37,780 DEBUG [hconnection-0x176b7d34-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-29T06:15:37,781 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:37086, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:15:37,782 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:37074, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:15:37,782 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=174, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:15:37,782 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:37102, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:15:37,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=174 2023-09-29T06:15:37,791 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:37108, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-29T06:15:37,791 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=174, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:15:37,791 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=175, ppid=174, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:15:37,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:37,796 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 0f1c71e3f887ce47e67c17dad51fd610 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2023-09-29T06:15:37,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=A 2023-09-29T06:15:37,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:37,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=B 2023-09-29T06:15:37,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:37,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=C 2023-09-29T06:15:37,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:37,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:37,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968197816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:37,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:37,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37034 deadline: 1695968197816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:37,818 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309293f0bad0956294872937c9c847bcb2001_0f1c71e3f887ce47e67c17dad51fd610 is 50, key is test_row_0/A:col10/1695968137792/Put/seqid=0 2023-09-29T06:15:37,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:37,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37074 deadline: 1695968197817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:37,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:37,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37058 deadline: 1695968197817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:37,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:37,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37086 deadline: 1695968197817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:37,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=174 2023-09-29T06:15:37,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:37,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968197918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:37,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:37,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37034 deadline: 1695968197918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:37,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:37,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37074 deadline: 1695968197920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:37,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:37,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37058 deadline: 1695968197920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:37,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:37,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37086 deadline: 1695968197920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:37,946 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2023-09-29T06:15:37,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:37,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. as already flushing 2023-09-29T06:15:37,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:37,946 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(53): pid=175 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:37,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:37,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:38,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=174 2023-09-29T06:15:38,098 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2023-09-29T06:15:38,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:38,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. as already flushing 2023-09-29T06:15:38,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:38,099 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(53): pid=175 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:38,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:38,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:38,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:38,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968198120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:38,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:38,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37034 deadline: 1695968198120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:38,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:38,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37086 deadline: 1695968198122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:38,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:38,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37074 deadline: 1695968198123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:38,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:38,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37058 deadline: 1695968198123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:38,240 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309293f0bad0956294872937c9c847bcb2001_0f1c71e3f887ce47e67c17dad51fd610 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309293f0bad0956294872937c9c847bcb2001_0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:38,241 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/a622c762a94d40f780e68341e1832272, store: [table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:38,241 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/a622c762a94d40f780e68341e1832272 is 175, key is test_row_0/A:col10/1695968137792/Put/seqid=0 2023-09-29T06:15:38,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2023-09-29T06:15:38,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:38,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. as already flushing 2023-09-29T06:15:38,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:38,251 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(53): pid=175 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:38,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:38,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:38,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=174 2023-09-29T06:15:38,404 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2023-09-29T06:15:38,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:38,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. as already flushing 2023-09-29T06:15:38,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:38,404 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(53): pid=175 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:38,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:38,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:38,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:38,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968198423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:38,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:38,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37034 deadline: 1695968198423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:38,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:38,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37074 deadline: 1695968198425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:38,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:38,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37058 deadline: 1695968198425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:38,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:38,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37086 deadline: 1695968198426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:38,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2023-09-29T06:15:38,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:38,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. as already flushing 2023-09-29T06:15:38,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:38,557 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(53): pid=175 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:38,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:38,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:38,645 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/a622c762a94d40f780e68341e1832272 2023-09-29T06:15:38,664 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/fbbbe3e374d94521ada24a6de6c2663d is 50, key is test_row_0/B:col10/1695968137792/Put/seqid=0 2023-09-29T06:15:38,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2023-09-29T06:15:38,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:38,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. as already flushing 2023-09-29T06:15:38,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:38,709 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(53): pid=175 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:38,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:38,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:38,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2023-09-29T06:15:38,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:38,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. as already flushing 2023-09-29T06:15:38,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:38,862 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(53): pid=175 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:38,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:38,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:38,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=174 2023-09-29T06:15:38,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:38,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968198925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:38,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:38,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37034 deadline: 1695968198927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:38,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:38,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37086 deadline: 1695968198929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:38,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:38,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37074 deadline: 1695968198929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:38,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:38,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37058 deadline: 1695968198930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:39,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2023-09-29T06:15:39,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:39,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. as already flushing 2023-09-29T06:15:39,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:39,014 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(53): pid=175 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:39,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:39,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:39,079 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/fbbbe3e374d94521ada24a6de6c2663d 2023-09-29T06:15:39,095 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/8461659db7624ed1a514a4352dc5db97 is 50, key is test_row_0/C:col10/1695968137792/Put/seqid=0 2023-09-29T06:15:39,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2023-09-29T06:15:39,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:39,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. as already flushing 2023-09-29T06:15:39,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:39,167 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(53): pid=175 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:39,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:39,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:39,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2023-09-29T06:15:39,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:39,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. as already flushing 2023-09-29T06:15:39,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:39,319 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(53): pid=175 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:39,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:39,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:39,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2023-09-29T06:15:39,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:39,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. as already flushing 2023-09-29T06:15:39,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:39,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(53): pid=175 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:39,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:39,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:39,499 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/8461659db7624ed1a514a4352dc5db97 2023-09-29T06:15:39,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/a622c762a94d40f780e68341e1832272 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/a622c762a94d40f780e68341e1832272 2023-09-29T06:15:39,507 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/a622c762a94d40f780e68341e1832272, entries=150, sequenceid=16, filesize=30.2 K 2023-09-29T06:15:39,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/fbbbe3e374d94521ada24a6de6c2663d as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/fbbbe3e374d94521ada24a6de6c2663d 2023-09-29T06:15:39,511 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/fbbbe3e374d94521ada24a6de6c2663d, entries=150, sequenceid=16, filesize=11.7 K 2023-09-29T06:15:39,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/8461659db7624ed1a514a4352dc5db97 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/8461659db7624ed1a514a4352dc5db97 2023-09-29T06:15:39,515 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/8461659db7624ed1a514a4352dc5db97, entries=150, sequenceid=16, filesize=11.7 K 2023-09-29T06:15:39,515 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 0f1c71e3f887ce47e67c17dad51fd610 in 1719ms, sequenceid=16, compaction requested=false 2023-09-29T06:15:39,516 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2023-09-29T06:15:39,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:39,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2023-09-29T06:15:39,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:39,624 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2819): Flushing 0f1c71e3f887ce47e67c17dad51fd610 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2023-09-29T06:15:39,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=A 2023-09-29T06:15:39,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:39,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=B 2023-09-29T06:15:39,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:39,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=C 2023-09-29T06:15:39,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:39,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929a78e13230b724511a035d6f36eb7958e_0f1c71e3f887ce47e67c17dad51fd610 is 50, key is test_row_0/A:col10/1695968137815/Put/seqid=0 2023-09-29T06:15:39,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=174 2023-09-29T06:15:39,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:39,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. as already flushing 2023-09-29T06:15:39,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:39,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:39,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37058 deadline: 1695968199936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:39,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37074 deadline: 1695968199935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:39,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:39,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37086 deadline: 1695968199936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:39,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:39,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968199938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:39,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:39,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37034 deadline: 1695968199938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:40,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:40,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37058 deadline: 1695968200039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:40,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:40,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37086 deadline: 1695968200039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:40,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:40,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37074 deadline: 1695968200039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:40,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:40,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968200041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:40,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:40,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37034 deadline: 1695968200041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:40,051 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929a78e13230b724511a035d6f36eb7958e_0f1c71e3f887ce47e67c17dad51fd610 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929a78e13230b724511a035d6f36eb7958e_0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:40,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/fd247fbbced14cb4943621fa4545acaa, store: [table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:40,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/fd247fbbced14cb4943621fa4545acaa is 175, key is test_row_0/A:col10/1695968137815/Put/seqid=0 2023-09-29T06:15:40,058 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/fd247fbbced14cb4943621fa4545acaa 2023-09-29T06:15:40,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/92e6bd729ea047f5a63c98d391b05472 is 50, key is test_row_0/B:col10/1695968137815/Put/seqid=0 2023-09-29T06:15:40,084 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2023-09-29T06:15:40,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:40,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37058 deadline: 1695968200241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:40,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:40,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37086 deadline: 1695968200242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:40,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:40,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37074 deadline: 1695968200242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:40,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:40,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968200242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:40,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:40,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37034 deadline: 1695968200244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:40,468 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/92e6bd729ea047f5a63c98d391b05472 2023-09-29T06:15:40,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/a9ccb23f36ca4a94a8af8bfc70110569 is 50, key is test_row_0/C:col10/1695968137815/Put/seqid=0 2023-09-29T06:15:40,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:40,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968200546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:40,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:40,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:40,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37086 deadline: 1695968200546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:40,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37058 deadline: 1695968200546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:40,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:40,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37074 deadline: 1695968200546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:40,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:40,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37034 deadline: 1695968200547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:40,879 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/a9ccb23f36ca4a94a8af8bfc70110569 2023-09-29T06:15:40,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/fd247fbbced14cb4943621fa4545acaa as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/fd247fbbced14cb4943621fa4545acaa 2023-09-29T06:15:40,886 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/fd247fbbced14cb4943621fa4545acaa, entries=150, sequenceid=41, filesize=30.2 K 2023-09-29T06:15:40,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/92e6bd729ea047f5a63c98d391b05472 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/92e6bd729ea047f5a63c98d391b05472 2023-09-29T06:15:40,890 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/92e6bd729ea047f5a63c98d391b05472, entries=150, sequenceid=41, filesize=11.7 K 2023-09-29T06:15:40,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/a9ccb23f36ca4a94a8af8bfc70110569 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/a9ccb23f36ca4a94a8af8bfc70110569 2023-09-29T06:15:40,893 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/a9ccb23f36ca4a94a8af8bfc70110569, entries=150, sequenceid=41, filesize=11.7 K 2023-09-29T06:15:40,894 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(3022): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 0f1c71e3f887ce47e67c17dad51fd610 in 1270ms, sequenceid=41, compaction requested=false 2023-09-29T06:15:40,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2520): Flush status journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:40,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:40,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=175 2023-09-29T06:15:40,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=175 2023-09-29T06:15:40,896 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=175, resume processing ppid=174 2023-09-29T06:15:40,896 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=175, ppid=174, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1040 sec 2023-09-29T06:15:40,897 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=174, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees in 3.1280 sec 2023-09-29T06:15:41,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:41,052 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 0f1c71e3f887ce47e67c17dad51fd610 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2023-09-29T06:15:41,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=A 2023-09-29T06:15:41,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:41,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=B 2023-09-29T06:15:41,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:41,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=C 2023-09-29T06:15:41,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:41,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309291489e7e936e94b73a6609d582671a15e_0f1c71e3f887ce47e67c17dad51fd610 is 50, key is test_row_0/A:col10/1695968141052/Put/seqid=0 2023-09-29T06:15:41,065 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309291489e7e936e94b73a6609d582671a15e_0f1c71e3f887ce47e67c17dad51fd610 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309291489e7e936e94b73a6609d582671a15e_0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:41,065 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/f8cb2c5594b84ff8ba125ba82c586b4d, store: [table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:41,066 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/f8cb2c5594b84ff8ba125ba82c586b4d is 175, key is test_row_0/A:col10/1695968141052/Put/seqid=0 2023-09-29T06:15:41,069 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/f8cb2c5594b84ff8ba125ba82c586b4d 2023-09-29T06:15:41,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:41,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968201066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:41,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:41,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37034 deadline: 1695968201067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:41,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:41,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37058 deadline: 1695968201068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:41,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:41,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37086 deadline: 1695968201070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:41,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:41,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37074 deadline: 1695968201070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:41,074 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/79341a55e74f4e0689d60bc302ade5fd is 50, key is test_row_0/B:col10/1695968141052/Put/seqid=0 2023-09-29T06:15:41,077 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/79341a55e74f4e0689d60bc302ade5fd 2023-09-29T06:15:41,081 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/9e04a716f69c47d1bb99aa134d2e67ab is 50, key is test_row_0/C:col10/1695968141052/Put/seqid=0 2023-09-29T06:15:41,084 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/9e04a716f69c47d1bb99aa134d2e67ab 2023-09-29T06:15:41,087 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/f8cb2c5594b84ff8ba125ba82c586b4d as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/f8cb2c5594b84ff8ba125ba82c586b4d 2023-09-29T06:15:41,090 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/f8cb2c5594b84ff8ba125ba82c586b4d, entries=150, sequenceid=54, filesize=30.2 K 2023-09-29T06:15:41,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/79341a55e74f4e0689d60bc302ade5fd as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/79341a55e74f4e0689d60bc302ade5fd 2023-09-29T06:15:41,093 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/79341a55e74f4e0689d60bc302ade5fd, entries=150, sequenceid=54, filesize=11.7 K 2023-09-29T06:15:41,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/9e04a716f69c47d1bb99aa134d2e67ab as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/9e04a716f69c47d1bb99aa134d2e67ab 2023-09-29T06:15:41,097 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/9e04a716f69c47d1bb99aa134d2e67ab, entries=150, sequenceid=54, filesize=11.7 K 2023-09-29T06:15:41,097 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 0f1c71e3f887ce47e67c17dad51fd610 in 46ms, sequenceid=54, compaction requested=true 2023-09-29T06:15:41,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:41,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f1c71e3f887ce47e67c17dad51fd610:A, priority=-2147483648, current under compaction store size is 1 2023-09-29T06:15:41,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:41,097 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:15:41,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f1c71e3f887ce47e67c17dad51fd610:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:15:41,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:41,097 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:15:41,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f1c71e3f887ce47e67c17dad51fd610:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:15:41,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:15:41,098 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92814 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:15:41,098 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35952 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:15:41,098 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 0f1c71e3f887ce47e67c17dad51fd610/A is initiating minor compaction (all files) 2023-09-29T06:15:41,098 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 0f1c71e3f887ce47e67c17dad51fd610/B is initiating minor compaction (all files) 2023-09-29T06:15:41,098 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 0f1c71e3f887ce47e67c17dad51fd610/B in TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:41,098 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 0f1c71e3f887ce47e67c17dad51fd610/A in TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:41,098 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/fbbbe3e374d94521ada24a6de6c2663d, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/92e6bd729ea047f5a63c98d391b05472, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/79341a55e74f4e0689d60bc302ade5fd] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp, totalSize=35.1 K 2023-09-29T06:15:41,098 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/a622c762a94d40f780e68341e1832272, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/fd247fbbced14cb4943621fa4545acaa, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/f8cb2c5594b84ff8ba125ba82c586b4d] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp, totalSize=90.6 K 2023-09-29T06:15:41,099 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:41,099 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. files: [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/a622c762a94d40f780e68341e1832272, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/fd247fbbced14cb4943621fa4545acaa, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/f8cb2c5594b84ff8ba125ba82c586b4d] 2023-09-29T06:15:41,099 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting fbbbe3e374d94521ada24a6de6c2663d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1695968137792 2023-09-29T06:15:41,099 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting a622c762a94d40f780e68341e1832272, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1695968137792 2023-09-29T06:15:41,099 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 92e6bd729ea047f5a63c98d391b05472, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1695968137815 2023-09-29T06:15:41,099 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting fd247fbbced14cb4943621fa4545acaa, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1695968137815 2023-09-29T06:15:41,100 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 79341a55e74f4e0689d60bc302ade5fd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1695968139937 2023-09-29T06:15:41,100 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting f8cb2c5594b84ff8ba125ba82c586b4d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1695968139937 2023-09-29T06:15:41,107 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f1c71e3f887ce47e67c17dad51fd610#B#compaction#557 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:41,108 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/e426edc6f3a043a4897c68127d7ca87f is 50, key is test_row_0/B:col10/1695968141052/Put/seqid=0 2023-09-29T06:15:41,109 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:41,110 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20230929748a7399212e4bfb83679382ae2c9059_0f1c71e3f887ce47e67c17dad51fd610 store=[table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:41,112 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20230929748a7399212e4bfb83679382ae2c9059_0f1c71e3f887ce47e67c17dad51fd610, store=[table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:41,112 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929748a7399212e4bfb83679382ae2c9059_0f1c71e3f887ce47e67c17dad51fd610 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:41,117 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/e426edc6f3a043a4897c68127d7ca87f as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/e426edc6f3a043a4897c68127d7ca87f 2023-09-29T06:15:41,121 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 0f1c71e3f887ce47e67c17dad51fd610/B of 0f1c71e3f887ce47e67c17dad51fd610 into e426edc6f3a043a4897c68127d7ca87f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:41,121 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:41,121 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610., storeName=0f1c71e3f887ce47e67c17dad51fd610/B, priority=13, startTime=1695968141097; duration=0sec 2023-09-29T06:15:41,121 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:15:41,121 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f1c71e3f887ce47e67c17dad51fd610:B 2023-09-29T06:15:41,121 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:15:41,122 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35952 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:15:41,122 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 0f1c71e3f887ce47e67c17dad51fd610/C is initiating minor compaction (all files) 2023-09-29T06:15:41,122 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 0f1c71e3f887ce47e67c17dad51fd610/C in TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:41,122 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/8461659db7624ed1a514a4352dc5db97, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/a9ccb23f36ca4a94a8af8bfc70110569, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/9e04a716f69c47d1bb99aa134d2e67ab] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp, totalSize=35.1 K 2023-09-29T06:15:41,123 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 8461659db7624ed1a514a4352dc5db97, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1695968137792 2023-09-29T06:15:41,123 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting a9ccb23f36ca4a94a8af8bfc70110569, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1695968137815 2023-09-29T06:15:41,123 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 9e04a716f69c47d1bb99aa134d2e67ab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1695968139937 2023-09-29T06:15:41,129 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f1c71e3f887ce47e67c17dad51fd610#C#compaction#559 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:41,130 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/664ed9a98df24e5e8313dd1f8525ce2a is 50, key is test_row_0/C:col10/1695968141052/Put/seqid=0 2023-09-29T06:15:41,136 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/664ed9a98df24e5e8313dd1f8525ce2a as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/664ed9a98df24e5e8313dd1f8525ce2a 2023-09-29T06:15:41,139 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 0f1c71e3f887ce47e67c17dad51fd610/C of 0f1c71e3f887ce47e67c17dad51fd610 into 664ed9a98df24e5e8313dd1f8525ce2a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:41,139 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:41,139 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610., storeName=0f1c71e3f887ce47e67c17dad51fd610/C, priority=13, startTime=1695968141097; duration=0sec 2023-09-29T06:15:41,139 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:41,139 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f1c71e3f887ce47e67c17dad51fd610:C 2023-09-29T06:15:41,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:41,173 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 0f1c71e3f887ce47e67c17dad51fd610 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2023-09-29T06:15:41,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=A 2023-09-29T06:15:41,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:41,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=B 2023-09-29T06:15:41,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:41,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=C 2023-09-29T06:15:41,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:41,177 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309292f0817b7819e4db1b64fd742e7e4a2dd_0f1c71e3f887ce47e67c17dad51fd610 is 50, key is test_row_0/A:col10/1695968141067/Put/seqid=0 2023-09-29T06:15:41,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:41,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37086 deadline: 1695968201177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:41,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:41,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37074 deadline: 1695968201178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:41,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:41,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37034 deadline: 1695968201179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:41,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:41,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37058 deadline: 1695968201179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:41,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:41,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968201180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:41,183 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309292f0817b7819e4db1b64fd742e7e4a2dd_0f1c71e3f887ce47e67c17dad51fd610 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309292f0817b7819e4db1b64fd742e7e4a2dd_0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:41,184 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/c48f39dd900a4714ab3fcebe81c4a5ef, store: [table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:41,185 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/c48f39dd900a4714ab3fcebe81c4a5ef is 175, key is test_row_0/A:col10/1695968141067/Put/seqid=0 2023-09-29T06:15:41,188 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=81, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/c48f39dd900a4714ab3fcebe81c4a5ef 2023-09-29T06:15:41,193 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/55e5b7fe941f49a494352663a96e0489 is 50, key is test_row_0/B:col10/1695968141067/Put/seqid=0 2023-09-29T06:15:41,196 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/55e5b7fe941f49a494352663a96e0489 2023-09-29T06:15:41,201 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/76722bf8fce641b2b1bbba24159556cf is 50, key is test_row_0/C:col10/1695968141067/Put/seqid=0 2023-09-29T06:15:41,205 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/76722bf8fce641b2b1bbba24159556cf 2023-09-29T06:15:41,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/c48f39dd900a4714ab3fcebe81c4a5ef as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/c48f39dd900a4714ab3fcebe81c4a5ef 2023-09-29T06:15:41,212 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/c48f39dd900a4714ab3fcebe81c4a5ef, entries=200, sequenceid=81, filesize=38.6 K 2023-09-29T06:15:41,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/55e5b7fe941f49a494352663a96e0489 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/55e5b7fe941f49a494352663a96e0489 2023-09-29T06:15:41,215 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/55e5b7fe941f49a494352663a96e0489, entries=150, sequenceid=81, filesize=11.7 K 2023-09-29T06:15:41,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/76722bf8fce641b2b1bbba24159556cf as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/76722bf8fce641b2b1bbba24159556cf 2023-09-29T06:15:41,220 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/76722bf8fce641b2b1bbba24159556cf, entries=150, sequenceid=81, filesize=11.7 K 2023-09-29T06:15:41,220 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 0f1c71e3f887ce47e67c17dad51fd610 in 47ms, sequenceid=81, compaction requested=false 2023-09-29T06:15:41,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:41,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:41,283 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 0f1c71e3f887ce47e67c17dad51fd610 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2023-09-29T06:15:41,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=A 2023-09-29T06:15:41,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:41,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=B 2023-09-29T06:15:41,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:41,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=C 2023-09-29T06:15:41,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:41,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309299ad3a68b38fb4d479d49a8e7c2f5ce8e_0f1c71e3f887ce47e67c17dad51fd610 is 50, key is test_row_0/A:col10/1695968141283/Put/seqid=0 2023-09-29T06:15:41,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:41,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37034 deadline: 1695968201321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:41,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:41,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37058 deadline: 1695968201323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:41,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:41,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37086 deadline: 1695968201325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:41,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:41,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968201326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:41,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:41,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37074 deadline: 1695968201326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:41,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:41,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37034 deadline: 1695968201427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:41,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:41,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37058 deadline: 1695968201427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:41,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:41,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37086 deadline: 1695968201429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:41,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:41,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968201430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:41,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:41,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37074 deadline: 1695968201430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:41,518 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f1c71e3f887ce47e67c17dad51fd610#A#compaction#558 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:41,518 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/7694fb17648442cf89a7ee43dd4d3912 is 175, key is test_row_0/A:col10/1695968141052/Put/seqid=0 2023-09-29T06:15:41,526 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/7694fb17648442cf89a7ee43dd4d3912 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/7694fb17648442cf89a7ee43dd4d3912 2023-09-29T06:15:41,530 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 0f1c71e3f887ce47e67c17dad51fd610/A of 0f1c71e3f887ce47e67c17dad51fd610 into 7694fb17648442cf89a7ee43dd4d3912(size=30.3 K), total size for store is 68.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:41,530 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:41,530 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610., storeName=0f1c71e3f887ce47e67c17dad51fd610/A, priority=13, startTime=1695968141097; duration=0sec 2023-09-29T06:15:41,530 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:41,530 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f1c71e3f887ce47e67c17dad51fd610:A 2023-09-29T06:15:41,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:41,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37034 deadline: 1695968201629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:41,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:41,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37058 deadline: 1695968201631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:41,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:41,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968201632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:41,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:41,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37086 deadline: 1695968201633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:41,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:41,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37074 deadline: 1695968201633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:41,697 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309299ad3a68b38fb4d479d49a8e7c2f5ce8e_0f1c71e3f887ce47e67c17dad51fd610 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309299ad3a68b38fb4d479d49a8e7c2f5ce8e_0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:41,698 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/945d293141324abeb220fa4c31756407, store: [table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:41,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/945d293141324abeb220fa4c31756407 is 175, key is test_row_0/A:col10/1695968141283/Put/seqid=0 2023-09-29T06:15:41,702 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=94, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/945d293141324abeb220fa4c31756407 2023-09-29T06:15:41,708 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/9a10fab4dc744666bc8bcd1574b03e7f is 50, key is test_row_0/B:col10/1695968141283/Put/seqid=0 2023-09-29T06:15:41,711 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/9a10fab4dc744666bc8bcd1574b03e7f 2023-09-29T06:15:41,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/f168994376664737966c0f92d02dde3f is 50, key is test_row_0/C:col10/1695968141283/Put/seqid=0 2023-09-29T06:15:41,719 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/f168994376664737966c0f92d02dde3f 2023-09-29T06:15:41,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/945d293141324abeb220fa4c31756407 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/945d293141324abeb220fa4c31756407 2023-09-29T06:15:41,726 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/945d293141324abeb220fa4c31756407, entries=150, sequenceid=94, filesize=30.2 K 2023-09-29T06:15:41,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/9a10fab4dc744666bc8bcd1574b03e7f as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/9a10fab4dc744666bc8bcd1574b03e7f 2023-09-29T06:15:41,730 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/9a10fab4dc744666bc8bcd1574b03e7f, entries=150, sequenceid=94, filesize=11.7 K 2023-09-29T06:15:41,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/f168994376664737966c0f92d02dde3f as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/f168994376664737966c0f92d02dde3f 2023-09-29T06:15:41,733 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/f168994376664737966c0f92d02dde3f, entries=150, sequenceid=94, filesize=11.7 K 2023-09-29T06:15:41,734 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 0f1c71e3f887ce47e67c17dad51fd610 in 451ms, sequenceid=94, compaction requested=true 2023-09-29T06:15:41,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:41,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f1c71e3f887ce47e67c17dad51fd610:A, priority=-2147483648, current under compaction store size is 1 2023-09-29T06:15:41,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:41,734 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:15:41,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f1c71e3f887ce47e67c17dad51fd610:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:15:41,734 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:15:41,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:41,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f1c71e3f887ce47e67c17dad51fd610:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:15:41,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:15:41,735 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101511 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:15:41,735 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 0f1c71e3f887ce47e67c17dad51fd610/A is initiating minor compaction (all files) 2023-09-29T06:15:41,735 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36055 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:15:41,735 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 0f1c71e3f887ce47e67c17dad51fd610/B is initiating minor compaction (all files) 2023-09-29T06:15:41,735 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 0f1c71e3f887ce47e67c17dad51fd610/A in TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:41,735 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 0f1c71e3f887ce47e67c17dad51fd610/B in TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:41,735 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/e426edc6f3a043a4897c68127d7ca87f, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/55e5b7fe941f49a494352663a96e0489, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/9a10fab4dc744666bc8bcd1574b03e7f] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp, totalSize=35.2 K 2023-09-29T06:15:41,735 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/7694fb17648442cf89a7ee43dd4d3912, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/c48f39dd900a4714ab3fcebe81c4a5ef, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/945d293141324abeb220fa4c31756407] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp, totalSize=99.1 K 2023-09-29T06:15:41,735 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:41,735 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. files: [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/7694fb17648442cf89a7ee43dd4d3912, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/c48f39dd900a4714ab3fcebe81c4a5ef, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/945d293141324abeb220fa4c31756407] 2023-09-29T06:15:41,736 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting e426edc6f3a043a4897c68127d7ca87f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1695968139937 2023-09-29T06:15:41,736 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 7694fb17648442cf89a7ee43dd4d3912, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1695968139937 2023-09-29T06:15:41,736 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 55e5b7fe941f49a494352663a96e0489, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1695968141065 2023-09-29T06:15:41,736 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting c48f39dd900a4714ab3fcebe81c4a5ef, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1695968141065 2023-09-29T06:15:41,736 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 9a10fab4dc744666bc8bcd1574b03e7f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1695968141178 2023-09-29T06:15:41,737 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 945d293141324abeb220fa4c31756407, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1695968141178 2023-09-29T06:15:41,742 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:41,743 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20230929828168028e8749bb954da3675afb88c4_0f1c71e3f887ce47e67c17dad51fd610 store=[table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:41,743 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f1c71e3f887ce47e67c17dad51fd610#B#compaction#567 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:41,744 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/483326bf15c14cc1a558c9a005468583 is 50, key is test_row_0/B:col10/1695968141283/Put/seqid=0 2023-09-29T06:15:41,744 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20230929828168028e8749bb954da3675afb88c4_0f1c71e3f887ce47e67c17dad51fd610, store=[table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:41,744 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929828168028e8749bb954da3675afb88c4_0f1c71e3f887ce47e67c17dad51fd610 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:41,752 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f1c71e3f887ce47e67c17dad51fd610#A#compaction#566 average throughput is 2.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:41,753 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/b26bc88f54fd484da3e4680f6bb4fa54 is 175, key is test_row_0/A:col10/1695968141283/Put/seqid=0 2023-09-29T06:15:41,766 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/b26bc88f54fd484da3e4680f6bb4fa54 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/b26bc88f54fd484da3e4680f6bb4fa54 2023-09-29T06:15:41,770 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 0f1c71e3f887ce47e67c17dad51fd610/A of 0f1c71e3f887ce47e67c17dad51fd610 into b26bc88f54fd484da3e4680f6bb4fa54(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:41,770 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:41,770 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610., storeName=0f1c71e3f887ce47e67c17dad51fd610/A, priority=13, startTime=1695968141734; duration=0sec 2023-09-29T06:15:41,770 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:15:41,770 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f1c71e3f887ce47e67c17dad51fd610:A 2023-09-29T06:15:41,770 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:15:41,771 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36055 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:15:41,772 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 0f1c71e3f887ce47e67c17dad51fd610/C is initiating minor compaction (all files) 2023-09-29T06:15:41,772 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 0f1c71e3f887ce47e67c17dad51fd610/C in TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:41,772 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/664ed9a98df24e5e8313dd1f8525ce2a, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/76722bf8fce641b2b1bbba24159556cf, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/f168994376664737966c0f92d02dde3f] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp, totalSize=35.2 K 2023-09-29T06:15:41,772 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 664ed9a98df24e5e8313dd1f8525ce2a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1695968139937 2023-09-29T06:15:41,772 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 76722bf8fce641b2b1bbba24159556cf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1695968141065 2023-09-29T06:15:41,773 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting f168994376664737966c0f92d02dde3f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1695968141178 2023-09-29T06:15:41,777 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f1c71e3f887ce47e67c17dad51fd610#C#compaction#568 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:41,778 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/b4a901048ab84f9bb07e6cbfbe5ca4f9 is 50, key is test_row_0/C:col10/1695968141283/Put/seqid=0 2023-09-29T06:15:41,785 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/b4a901048ab84f9bb07e6cbfbe5ca4f9 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/b4a901048ab84f9bb07e6cbfbe5ca4f9 2023-09-29T06:15:41,788 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 0f1c71e3f887ce47e67c17dad51fd610/C of 0f1c71e3f887ce47e67c17dad51fd610 into b4a901048ab84f9bb07e6cbfbe5ca4f9(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:41,788 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:41,788 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610., storeName=0f1c71e3f887ce47e67c17dad51fd610/C, priority=13, startTime=1695968141734; duration=0sec 2023-09-29T06:15:41,788 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:41,789 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f1c71e3f887ce47e67c17dad51fd610:C 2023-09-29T06:15:41,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=174 2023-09-29T06:15:41,895 INFO [Thread-3332 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 174 completed 2023-09-29T06:15:41,896 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:15:41,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=176, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees 2023-09-29T06:15:41,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=176 2023-09-29T06:15:41,899 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=176, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:15:41,899 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=176, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:15:41,899 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=177, ppid=176, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:15:41,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:41,933 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 0f1c71e3f887ce47e67c17dad51fd610 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2023-09-29T06:15:41,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=A 2023-09-29T06:15:41,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:41,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=B 2023-09-29T06:15:41,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:41,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=C 2023-09-29T06:15:41,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:41,940 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309299daee17f58a541fb9ef43dd1bd6d1f90_0f1c71e3f887ce47e67c17dad51fd610 is 50, key is test_row_0/A:col10/1695968141319/Put/seqid=0 2023-09-29T06:15:41,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:41,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968201941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:41,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:41,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37058 deadline: 1695968201941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:41,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:41,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:41,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37086 deadline: 1695968201941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:41,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37074 deadline: 1695968201942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:41,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:41,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37034 deadline: 1695968201942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:41,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=176 2023-09-29T06:15:42,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:42,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968202044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:42,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:42,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37058 deadline: 1695968202044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:42,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:42,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37086 deadline: 1695968202045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:42,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:42,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:42,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37074 deadline: 1695968202045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:42,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37034 deadline: 1695968202045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:42,051 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=177 2023-09-29T06:15:42,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:42,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. as already flushing 2023-09-29T06:15:42,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:42,052 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] handler.RSProcedureHandler(53): pid=177 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:42,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=177 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:42,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=177 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:42,156 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/483326bf15c14cc1a558c9a005468583 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/483326bf15c14cc1a558c9a005468583 2023-09-29T06:15:42,161 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 0f1c71e3f887ce47e67c17dad51fd610/B of 0f1c71e3f887ce47e67c17dad51fd610 into 483326bf15c14cc1a558c9a005468583(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:42,161 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:42,161 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610., storeName=0f1c71e3f887ce47e67c17dad51fd610/B, priority=13, startTime=1695968141734; duration=0sec 2023-09-29T06:15:42,161 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:42,161 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f1c71e3f887ce47e67c17dad51fd610:B 2023-09-29T06:15:42,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=176 2023-09-29T06:15:42,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=177 2023-09-29T06:15:42,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:42,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. as already flushing 2023-09-29T06:15:42,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:42,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] handler.RSProcedureHandler(53): pid=177 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:42,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=177 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:42,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=177 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:42,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:42,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968202247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:42,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:42,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:42,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37058 deadline: 1695968202247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:42,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37086 deadline: 1695968202247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:42,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:42,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:42,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37074 deadline: 1695968202247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:42,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37034 deadline: 1695968202247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:42,352 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309299daee17f58a541fb9ef43dd1bd6d1f90_0f1c71e3f887ce47e67c17dad51fd610 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309299daee17f58a541fb9ef43dd1bd6d1f90_0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:42,353 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/56f139aed7f644f3b045cd5efdba189f, store: [table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:42,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/56f139aed7f644f3b045cd5efdba189f is 175, key is test_row_0/A:col10/1695968141319/Put/seqid=0 2023-09-29T06:15:42,358 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=121, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/56f139aed7f644f3b045cd5efdba189f 2023-09-29T06:15:42,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/cd479ef3042144dc995a7918bc021098 is 50, key is test_row_0/B:col10/1695968141319/Put/seqid=0 2023-09-29T06:15:42,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=177 2023-09-29T06:15:42,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:42,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. as already flushing 2023-09-29T06:15:42,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:42,365 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] handler.RSProcedureHandler(53): pid=177 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:42,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=177 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:42,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=177 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:42,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=176 2023-09-29T06:15:42,517 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=177 2023-09-29T06:15:42,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:42,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. as already flushing 2023-09-29T06:15:42,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:42,518 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] handler.RSProcedureHandler(53): pid=177 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:42,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=177 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:42,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=177 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:42,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:42,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968202551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:42,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:42,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37086 deadline: 1695968202551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:42,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:42,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37058 deadline: 1695968202551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:42,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:42,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37034 deadline: 1695968202552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:42,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:42,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37074 deadline: 1695968202552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:42,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=177 2023-09-29T06:15:42,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:42,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. as already flushing 2023-09-29T06:15:42,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:42,670 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] handler.RSProcedureHandler(53): pid=177 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:42,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=177 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:42,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=177 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:42,775 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/cd479ef3042144dc995a7918bc021098 2023-09-29T06:15:42,780 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/94572942f9ed4009a6a5e489995919fd is 50, key is test_row_0/C:col10/1695968141319/Put/seqid=0 2023-09-29T06:15:42,786 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/94572942f9ed4009a6a5e489995919fd 2023-09-29T06:15:42,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/56f139aed7f644f3b045cd5efdba189f as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/56f139aed7f644f3b045cd5efdba189f 2023-09-29T06:15:42,793 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/56f139aed7f644f3b045cd5efdba189f, entries=200, sequenceid=121, filesize=38.6 K 2023-09-29T06:15:42,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/cd479ef3042144dc995a7918bc021098 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/cd479ef3042144dc995a7918bc021098 2023-09-29T06:15:42,797 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/cd479ef3042144dc995a7918bc021098, entries=150, sequenceid=121, filesize=11.7 K 2023-09-29T06:15:42,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/94572942f9ed4009a6a5e489995919fd as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/94572942f9ed4009a6a5e489995919fd 2023-09-29T06:15:42,800 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/94572942f9ed4009a6a5e489995919fd, entries=150, sequenceid=121, filesize=11.7 K 2023-09-29T06:15:42,801 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 0f1c71e3f887ce47e67c17dad51fd610 in 868ms, sequenceid=121, compaction requested=false 2023-09-29T06:15:42,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:42,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=177 2023-09-29T06:15:42,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:42,823 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2819): Flushing 0f1c71e3f887ce47e67c17dad51fd610 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2023-09-29T06:15:42,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=A 2023-09-29T06:15:42,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:42,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=B 2023-09-29T06:15:42,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:42,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=C 2023-09-29T06:15:42,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:42,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023092989cba9d8e3af4801a20e81d0e580e252_0f1c71e3f887ce47e67c17dad51fd610 is 50, key is test_row_0/A:col10/1695968141939/Put/seqid=0 2023-09-29T06:15:42,834 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023092989cba9d8e3af4801a20e81d0e580e252_0f1c71e3f887ce47e67c17dad51fd610 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023092989cba9d8e3af4801a20e81d0e580e252_0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:42,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/7fa34e860e2442c1ac8a8cd2ff5aec29, store: [table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:42,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/7fa34e860e2442c1ac8a8cd2ff5aec29 is 175, key is test_row_0/A:col10/1695968141939/Put/seqid=0 2023-09-29T06:15:43,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=176 2023-09-29T06:15:43,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:43,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. as already flushing 2023-09-29T06:15:43,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:43,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37086 deadline: 1695968203070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:43,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:43,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968203073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:43,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:43,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37074 deadline: 1695968203073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:43,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:43,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:43,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37034 deadline: 1695968203074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:43,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37058 deadline: 1695968203074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:43,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:43,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37086 deadline: 1695968203175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:43,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:43,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968203176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:43,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:43,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37074 deadline: 1695968203176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:43,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:43,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:43,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37034 deadline: 1695968203178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:43,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37058 deadline: 1695968203178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:43,239 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/7fa34e860e2442c1ac8a8cd2ff5aec29 2023-09-29T06:15:43,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/3e4e7ecdb8014740b85dd12f3313f99e is 50, key is test_row_0/B:col10/1695968141939/Put/seqid=0 2023-09-29T06:15:43,248 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/3e4e7ecdb8014740b85dd12f3313f99e 2023-09-29T06:15:43,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/4652fc5074d14c04bd7b977077b2a07d is 50, key is test_row_0/C:col10/1695968141939/Put/seqid=0 2023-09-29T06:15:43,256 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/4652fc5074d14c04bd7b977077b2a07d 2023-09-29T06:15:43,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/7fa34e860e2442c1ac8a8cd2ff5aec29 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/7fa34e860e2442c1ac8a8cd2ff5aec29 2023-09-29T06:15:43,264 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/7fa34e860e2442c1ac8a8cd2ff5aec29, entries=150, sequenceid=134, filesize=30.4 K 2023-09-29T06:15:43,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/3e4e7ecdb8014740b85dd12f3313f99e as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/3e4e7ecdb8014740b85dd12f3313f99e 2023-09-29T06:15:43,268 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/3e4e7ecdb8014740b85dd12f3313f99e, entries=150, sequenceid=134, filesize=11.8 K 2023-09-29T06:15:43,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/4652fc5074d14c04bd7b977077b2a07d as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/4652fc5074d14c04bd7b977077b2a07d 2023-09-29T06:15:43,272 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/4652fc5074d14c04bd7b977077b2a07d, entries=150, sequenceid=134, filesize=11.8 K 2023-09-29T06:15:43,272 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(3022): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 0f1c71e3f887ce47e67c17dad51fd610 in 450ms, sequenceid=134, compaction requested=true 2023-09-29T06:15:43,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2520): Flush status journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:43,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:43,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=177 2023-09-29T06:15:43,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=177 2023-09-29T06:15:43,274 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=177, resume processing ppid=176 2023-09-29T06:15:43,274 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=177, ppid=176, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3740 sec 2023-09-29T06:15:43,275 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=176, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees in 1.3790 sec 2023-09-29T06:15:43,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:43,380 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 0f1c71e3f887ce47e67c17dad51fd610 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2023-09-29T06:15:43,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=A 2023-09-29T06:15:43,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:43,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=B 2023-09-29T06:15:43,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:43,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=C 2023-09-29T06:15:43,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:43,385 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929535266286e624c629ad2f96cdb84f4d8_0f1c71e3f887ce47e67c17dad51fd610 is 50, key is test_row_0/A:col10/1695968143072/Put/seqid=0 2023-09-29T06:15:43,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:43,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37086 deadline: 1695968203383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:43,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:43,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:43,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37034 deadline: 1695968203384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:43,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37058 deadline: 1695968203385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:43,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:43,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968203386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:43,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:43,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37074 deadline: 1695968203387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:43,392 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929535266286e624c629ad2f96cdb84f4d8_0f1c71e3f887ce47e67c17dad51fd610 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929535266286e624c629ad2f96cdb84f4d8_0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:43,392 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/81b5ed1eea654b858e01ec80e7fe60c7, store: [table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:43,393 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/81b5ed1eea654b858e01ec80e7fe60c7 is 175, key is test_row_0/A:col10/1695968143072/Put/seqid=0 2023-09-29T06:15:43,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:43,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37086 deadline: 1695968203488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:43,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:43,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37034 deadline: 1695968203488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:43,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:43,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37058 deadline: 1695968203488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:43,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:43,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37074 deadline: 1695968203491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:43,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:43,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968203491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:43,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:43,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37034 deadline: 1695968203690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:43,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:43,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37086 deadline: 1695968203691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:43,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:43,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37058 deadline: 1695968203691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:43,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:43,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37074 deadline: 1695968203694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:43,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:43,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968203694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:43,796 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=160, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/81b5ed1eea654b858e01ec80e7fe60c7 2023-09-29T06:15:43,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/60f3e663bc4f47838fa5b5c9b9c86fde is 50, key is test_row_0/B:col10/1695968143072/Put/seqid=0 2023-09-29T06:15:43,806 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/60f3e663bc4f47838fa5b5c9b9c86fde 2023-09-29T06:15:43,814 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/f55aac84da7e4d76aff17c314da3130d is 50, key is test_row_0/C:col10/1695968143072/Put/seqid=0 2023-09-29T06:15:43,818 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/f55aac84da7e4d76aff17c314da3130d 2023-09-29T06:15:43,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/81b5ed1eea654b858e01ec80e7fe60c7 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/81b5ed1eea654b858e01ec80e7fe60c7 2023-09-29T06:15:43,825 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/81b5ed1eea654b858e01ec80e7fe60c7, entries=150, sequenceid=160, filesize=30.4 K 2023-09-29T06:15:43,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/60f3e663bc4f47838fa5b5c9b9c86fde as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/60f3e663bc4f47838fa5b5c9b9c86fde 2023-09-29T06:15:43,829 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/60f3e663bc4f47838fa5b5c9b9c86fde, entries=150, sequenceid=160, filesize=11.8 K 2023-09-29T06:15:43,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/f55aac84da7e4d76aff17c314da3130d as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/f55aac84da7e4d76aff17c314da3130d 2023-09-29T06:15:43,832 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/f55aac84da7e4d76aff17c314da3130d, entries=150, sequenceid=160, filesize=11.8 K 2023-09-29T06:15:43,833 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 0f1c71e3f887ce47e67c17dad51fd610 in 453ms, sequenceid=160, compaction requested=true 2023-09-29T06:15:43,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:43,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f1c71e3f887ce47e67c17dad51fd610:A, priority=-2147483648, current under compaction store size is 1 2023-09-29T06:15:43,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:43,833 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-29T06:15:43,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f1c71e3f887ce47e67c17dad51fd610:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:15:43,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:43,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f1c71e3f887ce47e67c17dad51fd610:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:15:43,833 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-29T06:15:43,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:15:43,834 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-29T06:15:43,834 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 0f1c71e3f887ce47e67c17dad51fd610/A is initiating minor compaction (all files) 2023-09-29T06:15:43,834 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 0f1c71e3f887ce47e67c17dad51fd610/A in TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:43,834 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48442 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-29T06:15:43,834 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/b26bc88f54fd484da3e4680f6bb4fa54, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/56f139aed7f644f3b045cd5efdba189f, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/7fa34e860e2442c1ac8a8cd2ff5aec29, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/81b5ed1eea654b858e01ec80e7fe60c7] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp, totalSize=129.7 K 2023-09-29T06:15:43,834 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:43,834 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 0f1c71e3f887ce47e67c17dad51fd610/B is initiating minor compaction (all files) 2023-09-29T06:15:43,834 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. files: [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/b26bc88f54fd484da3e4680f6bb4fa54, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/56f139aed7f644f3b045cd5efdba189f, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/7fa34e860e2442c1ac8a8cd2ff5aec29, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/81b5ed1eea654b858e01ec80e7fe60c7] 2023-09-29T06:15:43,834 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 0f1c71e3f887ce47e67c17dad51fd610/B in TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:43,835 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/483326bf15c14cc1a558c9a005468583, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/cd479ef3042144dc995a7918bc021098, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/3e4e7ecdb8014740b85dd12f3313f99e, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/60f3e663bc4f47838fa5b5c9b9c86fde] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp, totalSize=47.3 K 2023-09-29T06:15:43,835 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting b26bc88f54fd484da3e4680f6bb4fa54, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1695968141178 2023-09-29T06:15:43,835 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 483326bf15c14cc1a558c9a005468583, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1695968141178 2023-09-29T06:15:43,835 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 56f139aed7f644f3b045cd5efdba189f, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1695968141319 2023-09-29T06:15:43,835 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting cd479ef3042144dc995a7918bc021098, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1695968141319 2023-09-29T06:15:43,835 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 7fa34e860e2442c1ac8a8cd2ff5aec29, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1695968141939 2023-09-29T06:15:43,836 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 3e4e7ecdb8014740b85dd12f3313f99e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1695968141939 2023-09-29T06:15:43,836 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 81b5ed1eea654b858e01ec80e7fe60c7, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1695968143072 2023-09-29T06:15:43,836 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 60f3e663bc4f47838fa5b5c9b9c86fde, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1695968143072 2023-09-29T06:15:43,842 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:43,843 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20230929cc6edf109cab45a8a5a9fbcca575af49_0f1c71e3f887ce47e67c17dad51fd610 store=[table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:43,843 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f1c71e3f887ce47e67c17dad51fd610#B#compaction#578 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:43,844 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/1164583a760d4bf9aed78945ddbb82b6 is 50, key is test_row_0/B:col10/1695968143072/Put/seqid=0 2023-09-29T06:15:43,845 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20230929cc6edf109cab45a8a5a9fbcca575af49_0f1c71e3f887ce47e67c17dad51fd610, store=[table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:43,845 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929cc6edf109cab45a8a5a9fbcca575af49_0f1c71e3f887ce47e67c17dad51fd610 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:43,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:43,997 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 0f1c71e3f887ce47e67c17dad51fd610 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2023-09-29T06:15:43,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=A 2023-09-29T06:15:43,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:43,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=B 2023-09-29T06:15:43,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:43,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=C 2023-09-29T06:15:43,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:44,001 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929e5caab91836248caafbd178dd27eb743_0f1c71e3f887ce47e67c17dad51fd610 is 50, key is test_row_0/A:col10/1695968143385/Put/seqid=0 2023-09-29T06:15:44,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=176 2023-09-29T06:15:44,003 INFO [Thread-3332 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 176 completed 2023-09-29T06:15:44,004 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:15:44,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=178, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=178, table=TestAcidGuarantees 2023-09-29T06:15:44,006 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=178, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=178, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:15:44,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=178 2023-09-29T06:15:44,007 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=178, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=178, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:15:44,007 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=179, ppid=178, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:15:44,011 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929e5caab91836248caafbd178dd27eb743_0f1c71e3f887ce47e67c17dad51fd610 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929e5caab91836248caafbd178dd27eb743_0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:44,012 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/280b325d11fc4664a1a9b763be26b194, store: [table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:44,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/280b325d11fc4664a1a9b763be26b194 is 175, key is test_row_0/A:col10/1695968143385/Put/seqid=0 2023-09-29T06:15:44,016 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=171, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/280b325d11fc4664a1a9b763be26b194 2023-09-29T06:15:44,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/b7176e59f8f94c3db61fc08e1fd7d7a4 is 50, key is test_row_0/B:col10/1695968143385/Put/seqid=0 2023-09-29T06:15:44,025 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/b7176e59f8f94c3db61fc08e1fd7d7a4 2023-09-29T06:15:44,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:44,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968204022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:44,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:44,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37086 deadline: 1695968204023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:44,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:44,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37074 deadline: 1695968204024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:44,030 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/a19545db00904b188800ad44d04de9bf is 50, key is test_row_0/C:col10/1695968143385/Put/seqid=0 2023-09-29T06:15:44,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:44,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37058 deadline: 1695968204027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:44,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:44,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37034 deadline: 1695968204027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:44,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=178 2023-09-29T06:15:44,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:44,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37086 deadline: 1695968204128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:44,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:44,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37074 deadline: 1695968204128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:44,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:44,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968204128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:44,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:44,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37058 deadline: 1695968204132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:44,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:44,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37034 deadline: 1695968204132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:44,160 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=179 2023-09-29T06:15:44,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:44,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. as already flushing 2023-09-29T06:15:44,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:44,160 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] handler.RSProcedureHandler(53): pid=179 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:44,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=179 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:44,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=179 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:44,253 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f1c71e3f887ce47e67c17dad51fd610#A#compaction#579 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:44,254 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/b6b81bba26da41a9b3170540423796e7 is 175, key is test_row_0/A:col10/1695968143072/Put/seqid=0 2023-09-29T06:15:44,257 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/1164583a760d4bf9aed78945ddbb82b6 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/1164583a760d4bf9aed78945ddbb82b6 2023-09-29T06:15:44,261 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 0f1c71e3f887ce47e67c17dad51fd610/B of 0f1c71e3f887ce47e67c17dad51fd610 into 1164583a760d4bf9aed78945ddbb82b6(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:44,261 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:44,261 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610., storeName=0f1c71e3f887ce47e67c17dad51fd610/B, priority=12, startTime=1695968143833; duration=0sec 2023-09-29T06:15:44,261 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:15:44,261 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f1c71e3f887ce47e67c17dad51fd610:B 2023-09-29T06:15:44,261 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-29T06:15:44,262 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/b6b81bba26da41a9b3170540423796e7 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/b6b81bba26da41a9b3170540423796e7 2023-09-29T06:15:44,263 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48442 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-29T06:15:44,263 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 0f1c71e3f887ce47e67c17dad51fd610/C is initiating minor compaction (all files) 2023-09-29T06:15:44,263 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 0f1c71e3f887ce47e67c17dad51fd610/C in TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:44,263 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/b4a901048ab84f9bb07e6cbfbe5ca4f9, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/94572942f9ed4009a6a5e489995919fd, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/4652fc5074d14c04bd7b977077b2a07d, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/f55aac84da7e4d76aff17c314da3130d] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp, totalSize=47.3 K 2023-09-29T06:15:44,264 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting b4a901048ab84f9bb07e6cbfbe5ca4f9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1695968141178 2023-09-29T06:15:44,264 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 94572942f9ed4009a6a5e489995919fd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1695968141319 2023-09-29T06:15:44,264 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 4652fc5074d14c04bd7b977077b2a07d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1695968141939 2023-09-29T06:15:44,265 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting f55aac84da7e4d76aff17c314da3130d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1695968143072 2023-09-29T06:15:44,266 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 0f1c71e3f887ce47e67c17dad51fd610/A of 0f1c71e3f887ce47e67c17dad51fd610 into b6b81bba26da41a9b3170540423796e7(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:44,266 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:44,266 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610., storeName=0f1c71e3f887ce47e67c17dad51fd610/A, priority=12, startTime=1695968143833; duration=0sec 2023-09-29T06:15:44,266 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:44,266 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f1c71e3f887ce47e67c17dad51fd610:A 2023-09-29T06:15:44,271 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f1c71e3f887ce47e67c17dad51fd610#C#compaction#583 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:44,271 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/7ed4f6649311440d92235d85a8919148 is 50, key is test_row_0/C:col10/1695968143072/Put/seqid=0 2023-09-29T06:15:44,277 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/7ed4f6649311440d92235d85a8919148 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/7ed4f6649311440d92235d85a8919148 2023-09-29T06:15:44,282 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 0f1c71e3f887ce47e67c17dad51fd610/C of 0f1c71e3f887ce47e67c17dad51fd610 into 7ed4f6649311440d92235d85a8919148(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:44,282 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:44,282 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610., storeName=0f1c71e3f887ce47e67c17dad51fd610/C, priority=12, startTime=1695968143833; duration=0sec 2023-09-29T06:15:44,282 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:44,282 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f1c71e3f887ce47e67c17dad51fd610:C 2023-09-29T06:15:44,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=178 2023-09-29T06:15:44,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=179 2023-09-29T06:15:44,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:44,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. as already flushing 2023-09-29T06:15:44,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:44,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] handler.RSProcedureHandler(53): pid=179 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:44,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=179 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:44,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=179 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:44,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:44,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37074 deadline: 1695968204331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:44,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:44,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968204331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:44,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:44,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37086 deadline: 1695968204332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:44,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:44,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37058 deadline: 1695968204333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:44,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:44,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37034 deadline: 1695968204335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:44,433 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/a19545db00904b188800ad44d04de9bf 2023-09-29T06:15:44,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/280b325d11fc4664a1a9b763be26b194 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/280b325d11fc4664a1a9b763be26b194 2023-09-29T06:15:44,440 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/280b325d11fc4664a1a9b763be26b194, entries=150, sequenceid=171, filesize=30.4 K 2023-09-29T06:15:44,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/b7176e59f8f94c3db61fc08e1fd7d7a4 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/b7176e59f8f94c3db61fc08e1fd7d7a4 2023-09-29T06:15:44,444 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/b7176e59f8f94c3db61fc08e1fd7d7a4, entries=100, sequenceid=171, filesize=9.5 K 2023-09-29T06:15:44,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/a19545db00904b188800ad44d04de9bf as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/a19545db00904b188800ad44d04de9bf 2023-09-29T06:15:44,448 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/a19545db00904b188800ad44d04de9bf, entries=100, sequenceid=171, filesize=9.5 K 2023-09-29T06:15:44,448 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 0f1c71e3f887ce47e67c17dad51fd610 in 451ms, sequenceid=171, compaction requested=false 2023-09-29T06:15:44,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:44,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=179 2023-09-29T06:15:44,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:44,464 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2819): Flushing 0f1c71e3f887ce47e67c17dad51fd610 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2023-09-29T06:15:44,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=A 2023-09-29T06:15:44,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:44,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=B 2023-09-29T06:15:44,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:44,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=C 2023-09-29T06:15:44,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:44,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309296b88976b21444dcbb2e0306c0f0845ab_0f1c71e3f887ce47e67c17dad51fd610 is 50, key is test_row_0/A:col10/1695968144022/Put/seqid=0 2023-09-29T06:15:44,475 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309296b88976b21444dcbb2e0306c0f0845ab_0f1c71e3f887ce47e67c17dad51fd610 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309296b88976b21444dcbb2e0306c0f0845ab_0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:44,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/f89b4abed9dd48858f77fa28346352d4, store: [table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:44,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/f89b4abed9dd48858f77fa28346352d4 is 175, key is test_row_0/A:col10/1695968144022/Put/seqid=0 2023-09-29T06:15:44,480 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=199, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/f89b4abed9dd48858f77fa28346352d4 2023-09-29T06:15:44,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/8524bdbefe964c6db86a66eb265b7621 is 50, key is test_row_0/B:col10/1695968144022/Put/seqid=0 2023-09-29T06:15:44,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=178 2023-09-29T06:15:44,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. as already flushing 2023-09-29T06:15:44,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:44,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:44,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37058 deadline: 1695968204638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:44,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:44,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968204640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:44,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:44,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37086 deadline: 1695968204641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:44,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:44,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37074 deadline: 1695968204641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:44,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:44,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37034 deadline: 1695968204641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:44,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:44,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37058 deadline: 1695968204742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:44,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:44,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968204743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:44,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:44,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37086 deadline: 1695968204744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:44,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:44,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37034 deadline: 1695968204744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:44,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:44,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37074 deadline: 1695968204744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:44,891 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/8524bdbefe964c6db86a66eb265b7621 2023-09-29T06:15:44,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/9135b3c0464c4c61be979933f9aa89f5 is 50, key is test_row_0/C:col10/1695968144022/Put/seqid=0 2023-09-29T06:15:44,907 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/9135b3c0464c4c61be979933f9aa89f5 2023-09-29T06:15:44,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/f89b4abed9dd48858f77fa28346352d4 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/f89b4abed9dd48858f77fa28346352d4 2023-09-29T06:15:44,915 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/f89b4abed9dd48858f77fa28346352d4, entries=150, sequenceid=199, filesize=30.4 K 2023-09-29T06:15:44,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/8524bdbefe964c6db86a66eb265b7621 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/8524bdbefe964c6db86a66eb265b7621 2023-09-29T06:15:44,920 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/8524bdbefe964c6db86a66eb265b7621, entries=150, sequenceid=199, filesize=11.8 K 2023-09-29T06:15:44,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/9135b3c0464c4c61be979933f9aa89f5 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/9135b3c0464c4c61be979933f9aa89f5 2023-09-29T06:15:44,926 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/9135b3c0464c4c61be979933f9aa89f5, entries=150, sequenceid=199, filesize=11.8 K 2023-09-29T06:15:44,927 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(3022): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 0f1c71e3f887ce47e67c17dad51fd610 in 463ms, sequenceid=199, compaction requested=true 2023-09-29T06:15:44,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2520): Flush status journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:44,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:44,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=179 2023-09-29T06:15:44,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=179 2023-09-29T06:15:44,929 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=179, resume processing ppid=178 2023-09-29T06:15:44,929 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=179, ppid=178, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 921 msec 2023-09-29T06:15:44,930 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=178, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=178, table=TestAcidGuarantees in 925 msec 2023-09-29T06:15:44,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:44,948 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 0f1c71e3f887ce47e67c17dad51fd610 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2023-09-29T06:15:44,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=A 2023-09-29T06:15:44,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:44,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=B 2023-09-29T06:15:44,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:44,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=C 2023-09-29T06:15:44,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:44,955 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023092912acc22587314b089a5e0e4e48b0eaa6_0f1c71e3f887ce47e67c17dad51fd610 is 50, key is test_row_1/A:col10/1695968144947/Put/seqid=0 2023-09-29T06:15:44,961 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023092912acc22587314b089a5e0e4e48b0eaa6_0f1c71e3f887ce47e67c17dad51fd610 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023092912acc22587314b089a5e0e4e48b0eaa6_0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:44,961 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/21374984b6a441a2afba300510ad1b11, store: [table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:44,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/21374984b6a441a2afba300510ad1b11 is 175, key is test_row_1/A:col10/1695968144947/Put/seqid=0 2023-09-29T06:15:44,966 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=211, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/21374984b6a441a2afba300510ad1b11 2023-09-29T06:15:44,971 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/12d81dff835048848f8d0c554a90f88b is 50, key is test_row_1/B:col10/1695968144947/Put/seqid=0 2023-09-29T06:15:44,974 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/12d81dff835048848f8d0c554a90f88b 2023-09-29T06:15:44,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:44,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37074 deadline: 1695968204974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:44,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:44,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37034 deadline: 1695968204975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:44,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:44,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968204976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:44,980 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/30fc870dd8f94f058bfad7573205e596 is 50, key is test_row_1/C:col10/1695968144947/Put/seqid=0 2023-09-29T06:15:44,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:44,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37058 deadline: 1695968204978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:44,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:44,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37086 deadline: 1695968204978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:44,983 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/30fc870dd8f94f058bfad7573205e596 2023-09-29T06:15:44,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/21374984b6a441a2afba300510ad1b11 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/21374984b6a441a2afba300510ad1b11 2023-09-29T06:15:44,989 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/21374984b6a441a2afba300510ad1b11, entries=150, sequenceid=211, filesize=30.4 K 2023-09-29T06:15:44,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/12d81dff835048848f8d0c554a90f88b as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/12d81dff835048848f8d0c554a90f88b 2023-09-29T06:15:44,992 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/12d81dff835048848f8d0c554a90f88b, entries=150, sequenceid=211, filesize=11.8 K 2023-09-29T06:15:44,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/30fc870dd8f94f058bfad7573205e596 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/30fc870dd8f94f058bfad7573205e596 2023-09-29T06:15:44,996 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/30fc870dd8f94f058bfad7573205e596, entries=100, sequenceid=211, filesize=9.5 K 2023-09-29T06:15:44,997 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 0f1c71e3f887ce47e67c17dad51fd610 in 49ms, sequenceid=211, compaction requested=true 2023-09-29T06:15:44,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:44,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f1c71e3f887ce47e67c17dad51fd610:A, priority=-2147483648, current under compaction store size is 1 2023-09-29T06:15:44,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:44,997 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-29T06:15:44,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f1c71e3f887ce47e67c17dad51fd610:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:15:44,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:44,997 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-29T06:15:44,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f1c71e3f887ce47e67c17dad51fd610:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:15:44,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:15:44,999 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 124686 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-29T06:15:44,999 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 0f1c71e3f887ce47e67c17dad51fd610/A is initiating minor compaction (all files) 2023-09-29T06:15:44,999 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46480 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-29T06:15:44,999 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 0f1c71e3f887ce47e67c17dad51fd610/A in TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:44,999 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 0f1c71e3f887ce47e67c17dad51fd610/B is initiating minor compaction (all files) 2023-09-29T06:15:44,999 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/b6b81bba26da41a9b3170540423796e7, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/280b325d11fc4664a1a9b763be26b194, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/f89b4abed9dd48858f77fa28346352d4, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/21374984b6a441a2afba300510ad1b11] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp, totalSize=121.8 K 2023-09-29T06:15:44,999 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 0f1c71e3f887ce47e67c17dad51fd610/B in TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:44,999 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:44,999 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. files: [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/b6b81bba26da41a9b3170540423796e7, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/280b325d11fc4664a1a9b763be26b194, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/f89b4abed9dd48858f77fa28346352d4, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/21374984b6a441a2afba300510ad1b11] 2023-09-29T06:15:44,999 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/1164583a760d4bf9aed78945ddbb82b6, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/b7176e59f8f94c3db61fc08e1fd7d7a4, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/8524bdbefe964c6db86a66eb265b7621, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/12d81dff835048848f8d0c554a90f88b] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp, totalSize=45.4 K 2023-09-29T06:15:44,999 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 1164583a760d4bf9aed78945ddbb82b6, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1695968143072 2023-09-29T06:15:44,999 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting b6b81bba26da41a9b3170540423796e7, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1695968143072 2023-09-29T06:15:45,000 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting b7176e59f8f94c3db61fc08e1fd7d7a4, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1695968143385 2023-09-29T06:15:45,000 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 280b325d11fc4664a1a9b763be26b194, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1695968143385 2023-09-29T06:15:45,000 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 8524bdbefe964c6db86a66eb265b7621, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1695968144022 2023-09-29T06:15:45,000 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting f89b4abed9dd48858f77fa28346352d4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1695968144022 2023-09-29T06:15:45,000 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 12d81dff835048848f8d0c554a90f88b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1695968144639 2023-09-29T06:15:45,000 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 21374984b6a441a2afba300510ad1b11, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1695968144639 2023-09-29T06:15:45,006 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:45,006 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f1c71e3f887ce47e67c17dad51fd610#B#compaction#590 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:45,007 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/ec218e2d29754a7cbda1ee33800e6cfe is 50, key is test_row_0/B:col10/1695968144022/Put/seqid=0 2023-09-29T06:15:45,007 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20230929254e33bb02aa4772b103833948a13b95_0f1c71e3f887ce47e67c17dad51fd610 store=[table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:45,009 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20230929254e33bb02aa4772b103833948a13b95_0f1c71e3f887ce47e67c17dad51fd610, store=[table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:45,009 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929254e33bb02aa4772b103833948a13b95_0f1c71e3f887ce47e67c17dad51fd610 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:45,015 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/ec218e2d29754a7cbda1ee33800e6cfe as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/ec218e2d29754a7cbda1ee33800e6cfe 2023-09-29T06:15:45,019 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 0f1c71e3f887ce47e67c17dad51fd610/B of 0f1c71e3f887ce47e67c17dad51fd610 into ec218e2d29754a7cbda1ee33800e6cfe(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:45,019 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:45,019 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610., storeName=0f1c71e3f887ce47e67c17dad51fd610/B, priority=12, startTime=1695968144997; duration=0sec 2023-09-29T06:15:45,019 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:15:45,019 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f1c71e3f887ce47e67c17dad51fd610:B 2023-09-29T06:15:45,019 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-29T06:15:45,020 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 44090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-29T06:15:45,020 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 0f1c71e3f887ce47e67c17dad51fd610/C is initiating minor compaction (all files) 2023-09-29T06:15:45,020 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 0f1c71e3f887ce47e67c17dad51fd610/C in TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:45,020 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/7ed4f6649311440d92235d85a8919148, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/a19545db00904b188800ad44d04de9bf, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/9135b3c0464c4c61be979933f9aa89f5, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/30fc870dd8f94f058bfad7573205e596] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp, totalSize=43.1 K 2023-09-29T06:15:45,021 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 7ed4f6649311440d92235d85a8919148, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1695968143072 2023-09-29T06:15:45,021 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting a19545db00904b188800ad44d04de9bf, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1695968143385 2023-09-29T06:15:45,021 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 9135b3c0464c4c61be979933f9aa89f5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1695968144022 2023-09-29T06:15:45,021 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 30fc870dd8f94f058bfad7573205e596, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1695968144640 2023-09-29T06:15:45,027 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f1c71e3f887ce47e67c17dad51fd610#C#compaction#592 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:45,028 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/a6d3f9cd54ad4ac0bff1ac74556ad864 is 50, key is test_row_0/C:col10/1695968144022/Put/seqid=0 2023-09-29T06:15:45,035 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/a6d3f9cd54ad4ac0bff1ac74556ad864 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/a6d3f9cd54ad4ac0bff1ac74556ad864 2023-09-29T06:15:45,039 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 0f1c71e3f887ce47e67c17dad51fd610/C of 0f1c71e3f887ce47e67c17dad51fd610 into a6d3f9cd54ad4ac0bff1ac74556ad864(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:45,039 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:45,039 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610., storeName=0f1c71e3f887ce47e67c17dad51fd610/C, priority=12, startTime=1695968144997; duration=0sec 2023-09-29T06:15:45,039 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:45,039 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f1c71e3f887ce47e67c17dad51fd610:C 2023-09-29T06:15:45,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:45,082 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 0f1c71e3f887ce47e67c17dad51fd610 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2023-09-29T06:15:45,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=A 2023-09-29T06:15:45,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:45,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=B 2023-09-29T06:15:45,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:45,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=C 2023-09-29T06:15:45,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:45,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:45,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37058 deadline: 1695968205086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:45,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:45,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37086 deadline: 1695968205086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:45,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:45,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37034 deadline: 1695968205087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:45,089 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929a24bf84f6a3e4e6386936b0997e6b0a4_0f1c71e3f887ce47e67c17dad51fd610 is 50, key is test_row_0/A:col10/1695968144973/Put/seqid=0 2023-09-29T06:15:45,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:45,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37074 deadline: 1695968205087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:45,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:45,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968205087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:45,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=178 2023-09-29T06:15:45,110 INFO [Thread-3332 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 178 completed 2023-09-29T06:15:45,111 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:15:45,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=180, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=180, table=TestAcidGuarantees 2023-09-29T06:15:45,113 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=180, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=180, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:15:45,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=180 2023-09-29T06:15:45,114 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=180, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=180, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:15:45,114 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=181, ppid=180, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:15:45,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:45,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37058 deadline: 1695968205189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:45,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:45,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37086 deadline: 1695968205190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:45,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:45,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37034 deadline: 1695968205190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:45,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:45,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37074 deadline: 1695968205190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:45,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:45,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968205190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:45,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=180 2023-09-29T06:15:45,266 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2023-09-29T06:15:45,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:45,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. as already flushing 2023-09-29T06:15:45,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:45,267 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] handler.RSProcedureHandler(53): pid=181 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:45,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=181 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:45,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=181 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:45,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:45,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37058 deadline: 1695968205392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:45,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:45,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37086 deadline: 1695968205392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:45,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:45,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37074 deadline: 1695968205393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:45,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:45,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968205393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:45,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:45,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37034 deadline: 1695968205393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:45,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=180 2023-09-29T06:15:45,415 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f1c71e3f887ce47e67c17dad51fd610#A#compaction#591 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:45,416 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/51cc6b734dc34c06adcd2e56e0b5ba2b is 175, key is test_row_0/A:col10/1695968144022/Put/seqid=0 2023-09-29T06:15:45,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2023-09-29T06:15:45,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:45,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. as already flushing 2023-09-29T06:15:45,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:45,420 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] handler.RSProcedureHandler(53): pid=181 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:45,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=181 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:45,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=181 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:45,428 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/51cc6b734dc34c06adcd2e56e0b5ba2b as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/51cc6b734dc34c06adcd2e56e0b5ba2b 2023-09-29T06:15:45,431 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 0f1c71e3f887ce47e67c17dad51fd610/A of 0f1c71e3f887ce47e67c17dad51fd610 into 51cc6b734dc34c06adcd2e56e0b5ba2b(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:45,431 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:45,431 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610., storeName=0f1c71e3f887ce47e67c17dad51fd610/A, priority=12, startTime=1695968144997; duration=0sec 2023-09-29T06:15:45,431 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:45,432 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f1c71e3f887ce47e67c17dad51fd610:A 2023-09-29T06:15:45,496 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929a24bf84f6a3e4e6386936b0997e6b0a4_0f1c71e3f887ce47e67c17dad51fd610 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929a24bf84f6a3e4e6386936b0997e6b0a4_0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:45,496 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/206ce229acde432384e3898c2ddababd, store: [table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:45,497 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/206ce229acde432384e3898c2ddababd is 175, key is test_row_0/A:col10/1695968144973/Put/seqid=0 2023-09-29T06:15:45,500 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=241, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/206ce229acde432384e3898c2ddababd 2023-09-29T06:15:45,506 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/600c823c598543a9b74283f0d83671b3 is 50, key is test_row_0/B:col10/1695968144973/Put/seqid=0 2023-09-29T06:15:45,509 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/600c823c598543a9b74283f0d83671b3 2023-09-29T06:15:45,515 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/8896e151a20344ce9a20affce47ad963 is 50, key is test_row_0/C:col10/1695968144973/Put/seqid=0 2023-09-29T06:15:45,518 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/8896e151a20344ce9a20affce47ad963 2023-09-29T06:15:45,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/206ce229acde432384e3898c2ddababd as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/206ce229acde432384e3898c2ddababd 2023-09-29T06:15:45,525 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/206ce229acde432384e3898c2ddababd, entries=200, sequenceid=241, filesize=38.8 K 2023-09-29T06:15:45,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/600c823c598543a9b74283f0d83671b3 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/600c823c598543a9b74283f0d83671b3 2023-09-29T06:15:45,528 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/600c823c598543a9b74283f0d83671b3, entries=150, sequenceid=241, filesize=11.8 K 2023-09-29T06:15:45,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/8896e151a20344ce9a20affce47ad963 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/8896e151a20344ce9a20affce47ad963 2023-09-29T06:15:45,532 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/8896e151a20344ce9a20affce47ad963, entries=150, sequenceid=241, filesize=11.8 K 2023-09-29T06:15:45,533 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 0f1c71e3f887ce47e67c17dad51fd610 in 451ms, sequenceid=241, compaction requested=false 2023-09-29T06:15:45,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:45,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2023-09-29T06:15:45,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:45,572 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2819): Flushing 0f1c71e3f887ce47e67c17dad51fd610 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2023-09-29T06:15:45,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=A 2023-09-29T06:15:45,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:45,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=B 2023-09-29T06:15:45,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:45,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=C 2023-09-29T06:15:45,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:45,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929bc2e7d0761a84c1f867439dbc89f8611_0f1c71e3f887ce47e67c17dad51fd610 is 50, key is test_row_0/A:col10/1695968145086/Put/seqid=0 2023-09-29T06:15:45,583 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929bc2e7d0761a84c1f867439dbc89f8611_0f1c71e3f887ce47e67c17dad51fd610 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929bc2e7d0761a84c1f867439dbc89f8611_0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:45,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/11dd37f357cf4f948af57c27bb327125, store: [table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:45,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/11dd37f357cf4f948af57c27bb327125 is 175, key is test_row_0/A:col10/1695968145086/Put/seqid=0 2023-09-29T06:15:45,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:45,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. as already flushing 2023-09-29T06:15:45,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:45,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:45,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37086 deadline: 1695968205711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:45,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37058 deadline: 1695968205711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:45,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:45,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37034 deadline: 1695968205713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:45,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=180 2023-09-29T06:15:45,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:45,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37074 deadline: 1695968205714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:45,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:45,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968205714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:45,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:45,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37058 deadline: 1695968205815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:45,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:45,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:45,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37086 deadline: 1695968205815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:45,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37034 deadline: 1695968205815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:45,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:45,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37074 deadline: 1695968205817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:45,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:45,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968205817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:45,995 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=250, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/11dd37f357cf4f948af57c27bb327125 2023-09-29T06:15:46,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/d077d7557734444fa9375e8e5ced921f is 50, key is test_row_0/B:col10/1695968145086/Put/seqid=0 2023-09-29T06:15:46,004 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/d077d7557734444fa9375e8e5ced921f 2023-09-29T06:15:46,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/4b8309695b3d4470bba8117dbdea6970 is 50, key is test_row_0/C:col10/1695968145086/Put/seqid=0 2023-09-29T06:15:46,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:46,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37034 deadline: 1695968206018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:46,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:46,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37058 deadline: 1695968206019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:46,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:46,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37086 deadline: 1695968206019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:46,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:46,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37074 deadline: 1695968206020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:46,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:46,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968206021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:46,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=180 2023-09-29T06:15:46,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:46,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37058 deadline: 1695968206322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:46,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:46,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37034 deadline: 1695968206323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:46,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:46,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37086 deadline: 1695968206323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:46,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:46,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37074 deadline: 1695968206324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:46,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:46,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968206326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:46,412 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/4b8309695b3d4470bba8117dbdea6970 2023-09-29T06:15:46,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/11dd37f357cf4f948af57c27bb327125 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/11dd37f357cf4f948af57c27bb327125 2023-09-29T06:15:46,420 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/11dd37f357cf4f948af57c27bb327125, entries=100, sequenceid=250, filesize=21.9 K 2023-09-29T06:15:46,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/d077d7557734444fa9375e8e5ced921f as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/d077d7557734444fa9375e8e5ced921f 2023-09-29T06:15:46,424 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/d077d7557734444fa9375e8e5ced921f, entries=100, sequenceid=250, filesize=9.5 K 2023-09-29T06:15:46,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/4b8309695b3d4470bba8117dbdea6970 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/4b8309695b3d4470bba8117dbdea6970 2023-09-29T06:15:46,427 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/4b8309695b3d4470bba8117dbdea6970, entries=100, sequenceid=250, filesize=9.5 K 2023-09-29T06:15:46,428 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(3022): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for 0f1c71e3f887ce47e67c17dad51fd610 in 856ms, sequenceid=250, compaction requested=true 2023-09-29T06:15:46,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2520): Flush status journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:46,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:46,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=181 2023-09-29T06:15:46,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=181 2023-09-29T06:15:46,430 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=181, resume processing ppid=180 2023-09-29T06:15:46,430 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=181, ppid=180, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3150 sec 2023-09-29T06:15:46,431 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=180, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=180, table=TestAcidGuarantees in 1.3190 sec 2023-09-29T06:15:46,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:46,827 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 0f1c71e3f887ce47e67c17dad51fd610 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2023-09-29T06:15:46,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=A 2023-09-29T06:15:46,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:46,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=B 2023-09-29T06:15:46,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:46,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=C 2023-09-29T06:15:46,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:46,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:46,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37086 deadline: 1695968206829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:46,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:46,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37058 deadline: 1695968206830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:46,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:46,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37034 deadline: 1695968206831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:46,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:46,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37074 deadline: 1695968206831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:46,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:46,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929749424148688428694c6e2c61b027253_0f1c71e3f887ce47e67c17dad51fd610 is 50, key is test_row_0/A:col10/1695968145712/Put/seqid=0 2023-09-29T06:15:46,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968206833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:46,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:46,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37086 deadline: 1695968206932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:46,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:46,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37058 deadline: 1695968206934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:46,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:46,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37034 deadline: 1695968206934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:46,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:46,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37074 deadline: 1695968206935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:47,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:47,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37086 deadline: 1695968207136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:47,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:47,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37034 deadline: 1695968207137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:47,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:47,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37058 deadline: 1695968207137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:47,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:47,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37074 deadline: 1695968207137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:47,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=180 2023-09-29T06:15:47,218 INFO [Thread-3332 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 180 completed 2023-09-29T06:15:47,219 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:15:47,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=182, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=182, table=TestAcidGuarantees 2023-09-29T06:15:47,221 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=182, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=182, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:15:47,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=182 2023-09-29T06:15:47,222 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=182, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=182, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:15:47,222 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=183, ppid=182, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:15:47,241 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929749424148688428694c6e2c61b027253_0f1c71e3f887ce47e67c17dad51fd610 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929749424148688428694c6e2c61b027253_0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:47,242 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/59ece13175e9451985555188b15c9c35, store: [table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:47,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/59ece13175e9451985555188b15c9c35 is 175, key is test_row_0/A:col10/1695968145712/Put/seqid=0 2023-09-29T06:15:47,246 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=279, memsize=58.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/59ece13175e9451985555188b15c9c35 2023-09-29T06:15:47,252 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/a8f8afa8735e47fc9ba84f0364ab0533 is 50, key is test_row_0/B:col10/1695968145712/Put/seqid=0 2023-09-29T06:15:47,255 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/a8f8afa8735e47fc9ba84f0364ab0533 2023-09-29T06:15:47,260 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/d5c61c506034478ea7e5bb202bd6791b is 50, key is test_row_0/C:col10/1695968145712/Put/seqid=0 2023-09-29T06:15:47,279 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/d5c61c506034478ea7e5bb202bd6791b 2023-09-29T06:15:47,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/59ece13175e9451985555188b15c9c35 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/59ece13175e9451985555188b15c9c35 2023-09-29T06:15:47,288 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/59ece13175e9451985555188b15c9c35, entries=200, sequenceid=279, filesize=39.0 K 2023-09-29T06:15:47,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/a8f8afa8735e47fc9ba84f0364ab0533 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/a8f8afa8735e47fc9ba84f0364ab0533 2023-09-29T06:15:47,292 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/a8f8afa8735e47fc9ba84f0364ab0533, entries=150, sequenceid=279, filesize=12.0 K 2023-09-29T06:15:47,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/d5c61c506034478ea7e5bb202bd6791b as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/d5c61c506034478ea7e5bb202bd6791b 2023-09-29T06:15:47,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/d5c61c506034478ea7e5bb202bd6791b, entries=150, sequenceid=279, filesize=12.0 K 2023-09-29T06:15:47,297 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for 0f1c71e3f887ce47e67c17dad51fd610 in 470ms, sequenceid=279, compaction requested=true 2023-09-29T06:15:47,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:47,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f1c71e3f887ce47e67c17dad51fd610:A, priority=-2147483648, current under compaction store size is 1 2023-09-29T06:15:47,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:47,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f1c71e3f887ce47e67c17dad51fd610:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:15:47,297 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-29T06:15:47,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:47,297 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-29T06:15:47,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f1c71e3f887ce47e67c17dad51fd610:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:15:47,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:15:47,299 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133781 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-29T06:15:47,299 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 0f1c71e3f887ce47e67c17dad51fd610/A is initiating minor compaction (all files) 2023-09-29T06:15:47,299 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 0f1c71e3f887ce47e67c17dad51fd610/A in TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:47,299 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46770 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-29T06:15:47,299 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 0f1c71e3f887ce47e67c17dad51fd610/B is initiating minor compaction (all files) 2023-09-29T06:15:47,299 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/51cc6b734dc34c06adcd2e56e0b5ba2b, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/206ce229acde432384e3898c2ddababd, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/11dd37f357cf4f948af57c27bb327125, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/59ece13175e9451985555188b15c9c35] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp, totalSize=130.6 K 2023-09-29T06:15:47,299 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 0f1c71e3f887ce47e67c17dad51fd610/B in TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:47,299 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:47,299 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/ec218e2d29754a7cbda1ee33800e6cfe, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/600c823c598543a9b74283f0d83671b3, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/d077d7557734444fa9375e8e5ced921f, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/a8f8afa8735e47fc9ba84f0364ab0533] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp, totalSize=45.7 K 2023-09-29T06:15:47,299 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. files: [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/51cc6b734dc34c06adcd2e56e0b5ba2b, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/206ce229acde432384e3898c2ddababd, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/11dd37f357cf4f948af57c27bb327125, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/59ece13175e9451985555188b15c9c35] 2023-09-29T06:15:47,299 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting ec218e2d29754a7cbda1ee33800e6cfe, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1695968144022 2023-09-29T06:15:47,299 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 51cc6b734dc34c06adcd2e56e0b5ba2b, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1695968144022 2023-09-29T06:15:47,300 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 600c823c598543a9b74283f0d83671b3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1695968144973 2023-09-29T06:15:47,300 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 206ce229acde432384e3898c2ddababd, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1695968144973 2023-09-29T06:15:47,300 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting d077d7557734444fa9375e8e5ced921f, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1695968145086 2023-09-29T06:15:47,300 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 11dd37f357cf4f948af57c27bb327125, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1695968145086 2023-09-29T06:15:47,300 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting a8f8afa8735e47fc9ba84f0364ab0533, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1695968145712 2023-09-29T06:15:47,300 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 59ece13175e9451985555188b15c9c35, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1695968145712 2023-09-29T06:15:47,306 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:47,307 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f1c71e3f887ce47e67c17dad51fd610#B#compaction#602 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:47,307 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2023092956ff4ad5e8584f07b9cee0bf06c45d22_0f1c71e3f887ce47e67c17dad51fd610 store=[table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:47,307 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/da3fd91a881a4d7d99c3a326a87cdc74 is 50, key is test_row_0/B:col10/1695968145712/Put/seqid=0 2023-09-29T06:15:47,309 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2023092956ff4ad5e8584f07b9cee0bf06c45d22_0f1c71e3f887ce47e67c17dad51fd610, store=[table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:47,309 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023092956ff4ad5e8584f07b9cee0bf06c45d22_0f1c71e3f887ce47e67c17dad51fd610 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:47,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=182 2023-09-29T06:15:47,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=183 2023-09-29T06:15:47,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:47,375 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HRegion(2819): Flushing 0f1c71e3f887ce47e67c17dad51fd610 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2023-09-29T06:15:47,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=A 2023-09-29T06:15:47,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:47,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=B 2023-09-29T06:15:47,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:47,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=C 2023-09-29T06:15:47,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:47,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=183}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309298249c67605e94dfca83fa39c237ca6d4_0f1c71e3f887ce47e67c17dad51fd610 is 50, key is test_row_0/A:col10/1695968146830/Put/seqid=0 2023-09-29T06:15:47,395 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309298249c67605e94dfca83fa39c237ca6d4_0f1c71e3f887ce47e67c17dad51fd610 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309298249c67605e94dfca83fa39c237ca6d4_0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:47,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=183}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/79850abb8b50400dbd10538be38f4a6c, store: [table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:47,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=183}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/79850abb8b50400dbd10538be38f4a6c is 175, key is test_row_0/A:col10/1695968146830/Put/seqid=0 2023-09-29T06:15:47,399 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=183}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=286, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/79850abb8b50400dbd10538be38f4a6c 2023-09-29T06:15:47,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=183}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/23e9e8b7bff14a9b90f0571a168bea31 is 50, key is test_row_0/B:col10/1695968146830/Put/seqid=0 2023-09-29T06:15:47,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:47,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. as already flushing 2023-09-29T06:15:47,460 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2023-09-29T06:15:47,460 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2023-09-29T06:15:47,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:47,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37058 deadline: 1695968207469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:47,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:47,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37086 deadline: 1695968207471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:47,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:47,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37074 deadline: 1695968207472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:47,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:47,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37034 deadline: 1695968207473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:47,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=182 2023-09-29T06:15:47,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:47,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37058 deadline: 1695968207574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:47,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:47,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37086 deadline: 1695968207575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:47,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:47,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37074 deadline: 1695968207576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:47,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:47,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37034 deadline: 1695968207577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:47,714 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f1c71e3f887ce47e67c17dad51fd610#A#compaction#603 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:47,715 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/a40f66022f054d33a3317c37714ae2bc is 175, key is test_row_0/A:col10/1695968145712/Put/seqid=0 2023-09-29T06:15:47,718 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/da3fd91a881a4d7d99c3a326a87cdc74 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/da3fd91a881a4d7d99c3a326a87cdc74 2023-09-29T06:15:47,723 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 0f1c71e3f887ce47e67c17dad51fd610/B of 0f1c71e3f887ce47e67c17dad51fd610 into da3fd91a881a4d7d99c3a326a87cdc74(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:47,723 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:47,723 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610., storeName=0f1c71e3f887ce47e67c17dad51fd610/B, priority=12, startTime=1695968147297; duration=0sec 2023-09-29T06:15:47,723 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:15:47,723 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f1c71e3f887ce47e67c17dad51fd610:B 2023-09-29T06:15:47,723 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-29T06:15:47,724 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46770 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-29T06:15:47,724 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 0f1c71e3f887ce47e67c17dad51fd610/C is initiating minor compaction (all files) 2023-09-29T06:15:47,724 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 0f1c71e3f887ce47e67c17dad51fd610/C in TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:47,724 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/a6d3f9cd54ad4ac0bff1ac74556ad864, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/8896e151a20344ce9a20affce47ad963, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/4b8309695b3d4470bba8117dbdea6970, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/d5c61c506034478ea7e5bb202bd6791b] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp, totalSize=45.7 K 2023-09-29T06:15:47,725 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting a6d3f9cd54ad4ac0bff1ac74556ad864, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1695968144022 2023-09-29T06:15:47,725 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 8896e151a20344ce9a20affce47ad963, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1695968144973 2023-09-29T06:15:47,725 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 4b8309695b3d4470bba8117dbdea6970, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1695968145086 2023-09-29T06:15:47,725 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting d5c61c506034478ea7e5bb202bd6791b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1695968145712 2023-09-29T06:15:47,731 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f1c71e3f887ce47e67c17dad51fd610#C#compaction#606 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:47,731 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/d1974cf42b46469188e2455b0a1390a1 is 50, key is test_row_0/C:col10/1695968145712/Put/seqid=0 2023-09-29T06:15:47,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:47,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37058 deadline: 1695968207778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:47,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:47,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37086 deadline: 1695968207778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:47,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:47,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37034 deadline: 1695968207779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:47,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:47,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37074 deadline: 1695968207780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:47,808 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/23e9e8b7bff14a9b90f0571a168bea31 2023-09-29T06:15:47,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=183}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/2871c6d06c394836831f2f4bcbf500b3 is 50, key is test_row_0/C:col10/1695968146830/Put/seqid=0 2023-09-29T06:15:47,820 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/2871c6d06c394836831f2f4bcbf500b3 2023-09-29T06:15:47,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=182 2023-09-29T06:15:47,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/79850abb8b50400dbd10538be38f4a6c as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/79850abb8b50400dbd10538be38f4a6c 2023-09-29T06:15:47,828 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/79850abb8b50400dbd10538be38f4a6c, entries=100, sequenceid=286, filesize=22.0 K 2023-09-29T06:15:47,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/23e9e8b7bff14a9b90f0571a168bea31 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/23e9e8b7bff14a9b90f0571a168bea31 2023-09-29T06:15:47,832 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/23e9e8b7bff14a9b90f0571a168bea31, entries=100, sequenceid=286, filesize=9.6 K 2023-09-29T06:15:47,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/2871c6d06c394836831f2f4bcbf500b3 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/2871c6d06c394836831f2f4bcbf500b3 2023-09-29T06:15:47,836 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/2871c6d06c394836831f2f4bcbf500b3, entries=100, sequenceid=286, filesize=9.6 K 2023-09-29T06:15:47,836 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HRegion(3022): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=181.14 KB/185490 for 0f1c71e3f887ce47e67c17dad51fd610 in 461ms, sequenceid=286, compaction requested=false 2023-09-29T06:15:47,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HRegion(2520): Flush status journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:47,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:47,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=183 2023-09-29T06:15:47,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=183 2023-09-29T06:15:47,838 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=183, resume processing ppid=182 2023-09-29T06:15:47,838 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=183, ppid=182, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 615 msec 2023-09-29T06:15:47,839 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=182, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=182, table=TestAcidGuarantees in 620 msec 2023-09-29T06:15:47,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:47,845 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 0f1c71e3f887ce47e67c17dad51fd610 3/3 column families, dataSize=187.85 KB heapSize=492.94 KB 2023-09-29T06:15:47,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=A 2023-09-29T06:15:47,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:47,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=B 2023-09-29T06:15:47,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:47,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=C 2023-09-29T06:15:47,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:47,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023092900e70be28e5d4aec9a04c7b434b03d15_0f1c71e3f887ce47e67c17dad51fd610 is 50, key is test_row_0/A:col10/1695968147844/Put/seqid=0 2023-09-29T06:15:47,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:47,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968207851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:47,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:47,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968207954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:48,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:48,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37058 deadline: 1695968208079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:48,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:48,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37086 deadline: 1695968208079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:48,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:48,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37034 deadline: 1695968208083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:48,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:48,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37074 deadline: 1695968208084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:48,123 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/a40f66022f054d33a3317c37714ae2bc as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/a40f66022f054d33a3317c37714ae2bc 2023-09-29T06:15:48,127 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 0f1c71e3f887ce47e67c17dad51fd610/A of 0f1c71e3f887ce47e67c17dad51fd610 into a40f66022f054d33a3317c37714ae2bc(size=31.1 K), total size for store is 53.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:48,127 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:48,128 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610., storeName=0f1c71e3f887ce47e67c17dad51fd610/A, priority=12, startTime=1695968147297; duration=0sec 2023-09-29T06:15:48,128 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:48,128 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f1c71e3f887ce47e67c17dad51fd610:A 2023-09-29T06:15:48,138 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/d1974cf42b46469188e2455b0a1390a1 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/d1974cf42b46469188e2455b0a1390a1 2023-09-29T06:15:48,142 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 0f1c71e3f887ce47e67c17dad51fd610/C of 0f1c71e3f887ce47e67c17dad51fd610 into d1974cf42b46469188e2455b0a1390a1(size=12.6 K), total size for store is 22.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:48,142 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:48,142 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610., storeName=0f1c71e3f887ce47e67c17dad51fd610/C, priority=12, startTime=1695968147297; duration=0sec 2023-09-29T06:15:48,142 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:48,143 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f1c71e3f887ce47e67c17dad51fd610:C 2023-09-29T06:15:48,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:48,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968208156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:48,256 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023092900e70be28e5d4aec9a04c7b434b03d15_0f1c71e3f887ce47e67c17dad51fd610 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023092900e70be28e5d4aec9a04c7b434b03d15_0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:48,257 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/c1eff5b143814c51aefdcebaa1c865c8, store: [table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:48,258 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/c1eff5b143814c51aefdcebaa1c865c8 is 175, key is test_row_0/A:col10/1695968147844/Put/seqid=0 2023-09-29T06:15:48,261 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=318, memsize=62.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/c1eff5b143814c51aefdcebaa1c865c8 2023-09-29T06:15:48,267 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/69361123921b4d6ea5c573afe22b211c is 50, key is test_row_0/B:col10/1695968147844/Put/seqid=0 2023-09-29T06:15:48,271 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/69361123921b4d6ea5c573afe22b211c 2023-09-29T06:15:48,276 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/093a2aea215446ffb95834425edcd8a9 is 50, key is test_row_0/C:col10/1695968147844/Put/seqid=0 2023-09-29T06:15:48,282 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/093a2aea215446ffb95834425edcd8a9 2023-09-29T06:15:48,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/c1eff5b143814c51aefdcebaa1c865c8 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/c1eff5b143814c51aefdcebaa1c865c8 2023-09-29T06:15:48,288 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/c1eff5b143814c51aefdcebaa1c865c8, entries=200, sequenceid=318, filesize=39.0 K 2023-09-29T06:15:48,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/69361123921b4d6ea5c573afe22b211c as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/69361123921b4d6ea5c573afe22b211c 2023-09-29T06:15:48,292 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/69361123921b4d6ea5c573afe22b211c, entries=150, sequenceid=318, filesize=12.0 K 2023-09-29T06:15:48,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/093a2aea215446ffb95834425edcd8a9 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/093a2aea215446ffb95834425edcd8a9 2023-09-29T06:15:48,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/093a2aea215446ffb95834425edcd8a9, entries=150, sequenceid=318, filesize=12.0 K 2023-09-29T06:15:48,297 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~187.85 KB/192360, heapSize ~492.89 KB/504720, currentSize=13.42 KB/13740 for 0f1c71e3f887ce47e67c17dad51fd610 in 452ms, sequenceid=318, compaction requested=true 2023-09-29T06:15:48,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:48,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f1c71e3f887ce47e67c17dad51fd610:A, priority=-2147483648, current under compaction store size is 1 2023-09-29T06:15:48,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:48,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f1c71e3f887ce47e67c17dad51fd610:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:15:48,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:48,297 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:15:48,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f1c71e3f887ce47e67c17dad51fd610:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:15:48,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:15:48,297 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:15:48,298 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94328 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:15:48,298 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 0f1c71e3f887ce47e67c17dad51fd610/A is initiating minor compaction (all files) 2023-09-29T06:15:48,298 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 0f1c71e3f887ce47e67c17dad51fd610/A in TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:48,298 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/a40f66022f054d33a3317c37714ae2bc, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/79850abb8b50400dbd10538be38f4a6c, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/c1eff5b143814c51aefdcebaa1c865c8] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp, totalSize=92.1 K 2023-09-29T06:15:48,298 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:48,298 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. files: [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/a40f66022f054d33a3317c37714ae2bc, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/79850abb8b50400dbd10538be38f4a6c, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/c1eff5b143814c51aefdcebaa1c865c8] 2023-09-29T06:15:48,298 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35022 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:15:48,299 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 0f1c71e3f887ce47e67c17dad51fd610/B is initiating minor compaction (all files) 2023-09-29T06:15:48,299 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 0f1c71e3f887ce47e67c17dad51fd610/B in TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:48,299 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/da3fd91a881a4d7d99c3a326a87cdc74, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/23e9e8b7bff14a9b90f0571a168bea31, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/69361123921b4d6ea5c573afe22b211c] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp, totalSize=34.2 K 2023-09-29T06:15:48,299 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting a40f66022f054d33a3317c37714ae2bc, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1695968145712 2023-09-29T06:15:48,299 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting da3fd91a881a4d7d99c3a326a87cdc74, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1695968145712 2023-09-29T06:15:48,299 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 79850abb8b50400dbd10538be38f4a6c, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1695968146829 2023-09-29T06:15:48,299 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 23e9e8b7bff14a9b90f0571a168bea31, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1695968146829 2023-09-29T06:15:48,300 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting c1eff5b143814c51aefdcebaa1c865c8, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1695968147466 2023-09-29T06:15:48,300 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 69361123921b4d6ea5c573afe22b211c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1695968147470 2023-09-29T06:15:48,305 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:48,306 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202309295ed7dc47beea46f08e3539c273affa90_0f1c71e3f887ce47e67c17dad51fd610 store=[table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:48,307 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f1c71e3f887ce47e67c17dad51fd610#B#compaction#612 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:48,308 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/6d6326409315494fa12fdca35314d343 is 50, key is test_row_0/B:col10/1695968147844/Put/seqid=0 2023-09-29T06:15:48,308 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202309295ed7dc47beea46f08e3539c273affa90_0f1c71e3f887ce47e67c17dad51fd610, store=[table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:48,308 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309295ed7dc47beea46f08e3539c273affa90_0f1c71e3f887ce47e67c17dad51fd610 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:48,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=182 2023-09-29T06:15:48,325 INFO [Thread-3332 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 182 completed 2023-09-29T06:15:48,326 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:15:48,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=184, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=184, table=TestAcidGuarantees 2023-09-29T06:15:48,328 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=184, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=184, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:15:48,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=184 2023-09-29T06:15:48,329 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=184, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=184, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:15:48,329 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=185, ppid=184, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:15:48,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=184 2023-09-29T06:15:48,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:48,475 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 0f1c71e3f887ce47e67c17dad51fd610 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2023-09-29T06:15:48,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=A 2023-09-29T06:15:48,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:48,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=B 2023-09-29T06:15:48,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:48,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=C 2023-09-29T06:15:48,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:48,481 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929a23ecc42090c4e5a90d9c0a82e6c62a4_0f1c71e3f887ce47e67c17dad51fd610 is 50, key is test_row_0/A:col10/1695968148471/Put/seqid=0 2023-09-29T06:15:48,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=185 2023-09-29T06:15:48,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:48,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. as already flushing 2023-09-29T06:15:48,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:48,482 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] handler.RSProcedureHandler(53): pid=185 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:48,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=185 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:48,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=185 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:48,489 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929a23ecc42090c4e5a90d9c0a82e6c62a4_0f1c71e3f887ce47e67c17dad51fd610 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929a23ecc42090c4e5a90d9c0a82e6c62a4_0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:48,490 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/b9144deae82e4bf9a5c01407faeb0c7c, store: [table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:48,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/b9144deae82e4bf9a5c01407faeb0c7c is 175, key is test_row_0/A:col10/1695968148471/Put/seqid=0 2023-09-29T06:15:48,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:48,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968208544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:48,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:48,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37058 deadline: 1695968208582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:48,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:48,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37086 deadline: 1695968208585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:48,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:48,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37074 deadline: 1695968208587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:48,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:48,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37034 deadline: 1695968208590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:48,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=184 2023-09-29T06:15:48,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=185 2023-09-29T06:15:48,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:48,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. as already flushing 2023-09-29T06:15:48,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:48,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] handler.RSProcedureHandler(53): pid=185 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:48,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=185 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:48,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=185 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:48,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:48,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968208647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:48,721 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f1c71e3f887ce47e67c17dad51fd610#A#compaction#611 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:48,722 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/37a13dccf6334906a4b33a3a47fcf33e is 175, key is test_row_0/A:col10/1695968147844/Put/seqid=0 2023-09-29T06:15:48,724 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/6d6326409315494fa12fdca35314d343 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/6d6326409315494fa12fdca35314d343 2023-09-29T06:15:48,729 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 0f1c71e3f887ce47e67c17dad51fd610/B of 0f1c71e3f887ce47e67c17dad51fd610 into 6d6326409315494fa12fdca35314d343(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:48,729 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:48,729 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610., storeName=0f1c71e3f887ce47e67c17dad51fd610/B, priority=13, startTime=1695968148297; duration=0sec 2023-09-29T06:15:48,729 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:15:48,729 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f1c71e3f887ce47e67c17dad51fd610:B 2023-09-29T06:15:48,729 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/37a13dccf6334906a4b33a3a47fcf33e as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/37a13dccf6334906a4b33a3a47fcf33e 2023-09-29T06:15:48,729 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:15:48,730 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35022 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:15:48,730 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 0f1c71e3f887ce47e67c17dad51fd610/C is initiating minor compaction (all files) 2023-09-29T06:15:48,730 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 0f1c71e3f887ce47e67c17dad51fd610/C in TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:48,730 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/d1974cf42b46469188e2455b0a1390a1, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/2871c6d06c394836831f2f4bcbf500b3, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/093a2aea215446ffb95834425edcd8a9] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp, totalSize=34.2 K 2023-09-29T06:15:48,731 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting d1974cf42b46469188e2455b0a1390a1, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1695968145712 2023-09-29T06:15:48,731 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 2871c6d06c394836831f2f4bcbf500b3, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1695968146829 2023-09-29T06:15:48,731 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 093a2aea215446ffb95834425edcd8a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1695968147470 2023-09-29T06:15:48,733 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 0f1c71e3f887ce47e67c17dad51fd610/A of 0f1c71e3f887ce47e67c17dad51fd610 into 37a13dccf6334906a4b33a3a47fcf33e(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:48,733 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:48,733 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610., storeName=0f1c71e3f887ce47e67c17dad51fd610/A, priority=13, startTime=1695968148297; duration=0sec 2023-09-29T06:15:48,733 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:48,733 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f1c71e3f887ce47e67c17dad51fd610:A 2023-09-29T06:15:48,737 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f1c71e3f887ce47e67c17dad51fd610#C#compaction#614 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:48,737 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/2726e80b875c4209a4d12a97a1dbbc90 is 50, key is test_row_0/C:col10/1695968147844/Put/seqid=0 2023-09-29T06:15:48,745 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/2726e80b875c4209a4d12a97a1dbbc90 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/2726e80b875c4209a4d12a97a1dbbc90 2023-09-29T06:15:48,748 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 0f1c71e3f887ce47e67c17dad51fd610/C of 0f1c71e3f887ce47e67c17dad51fd610 into 2726e80b875c4209a4d12a97a1dbbc90(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:48,748 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:48,748 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610., storeName=0f1c71e3f887ce47e67c17dad51fd610/C, priority=13, startTime=1695968148297; duration=0sec 2023-09-29T06:15:48,748 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:48,748 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f1c71e3f887ce47e67c17dad51fd610:C 2023-09-29T06:15:48,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=185 2023-09-29T06:15:48,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:48,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. as already flushing 2023-09-29T06:15:48,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:48,787 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] handler.RSProcedureHandler(53): pid=185 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:48,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=185 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:48,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=185 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:48,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:48,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968208850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:48,892 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=331, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/b9144deae82e4bf9a5c01407faeb0c7c 2023-09-29T06:15:48,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/913626044c0b4676992d400b1e0e1c41 is 50, key is test_row_0/B:col10/1695968148471/Put/seqid=0 2023-09-29T06:15:48,902 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/913626044c0b4676992d400b1e0e1c41 2023-09-29T06:15:48,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/b6df0ef94a144f268e79f1b1586a09cd is 50, key is test_row_0/C:col10/1695968148471/Put/seqid=0 2023-09-29T06:15:48,910 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/b6df0ef94a144f268e79f1b1586a09cd 2023-09-29T06:15:48,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/b9144deae82e4bf9a5c01407faeb0c7c as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/b9144deae82e4bf9a5c01407faeb0c7c 2023-09-29T06:15:48,916 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/b9144deae82e4bf9a5c01407faeb0c7c, entries=400, sequenceid=331, filesize=73.0 K 2023-09-29T06:15:48,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/913626044c0b4676992d400b1e0e1c41 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/913626044c0b4676992d400b1e0e1c41 2023-09-29T06:15:48,920 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/913626044c0b4676992d400b1e0e1c41, entries=150, sequenceid=331, filesize=12.0 K 2023-09-29T06:15:48,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/b6df0ef94a144f268e79f1b1586a09cd as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/b6df0ef94a144f268e79f1b1586a09cd 2023-09-29T06:15:48,924 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/b6df0ef94a144f268e79f1b1586a09cd, entries=150, sequenceid=331, filesize=12.0 K 2023-09-29T06:15:48,924 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 0f1c71e3f887ce47e67c17dad51fd610 in 449ms, sequenceid=331, compaction requested=false 2023-09-29T06:15:48,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:48,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=184 2023-09-29T06:15:48,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=185 2023-09-29T06:15:48,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:48,939 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HRegion(2819): Flushing 0f1c71e3f887ce47e67c17dad51fd610 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2023-09-29T06:15:48,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=A 2023-09-29T06:15:48,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:48,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=B 2023-09-29T06:15:48,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:48,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=C 2023-09-29T06:15:48,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:48,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309290f8ab063fc1541c6a1629ceaf2a53b8a_0f1c71e3f887ce47e67c17dad51fd610 is 50, key is test_row_0/A:col10/1695968148529/Put/seqid=0 2023-09-29T06:15:49,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:49,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. as already flushing 2023-09-29T06:15:49,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:49,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968209173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:49,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:49,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968209275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:49,351 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309290f8ab063fc1541c6a1629ceaf2a53b8a_0f1c71e3f887ce47e67c17dad51fd610 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309290f8ab063fc1541c6a1629ceaf2a53b8a_0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:49,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/a11a931452fb4e4a993c01a9844c8eec, store: [table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:49,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/a11a931452fb4e4a993c01a9844c8eec is 175, key is test_row_0/A:col10/1695968148529/Put/seqid=0 2023-09-29T06:15:49,357 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=359, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/a11a931452fb4e4a993c01a9844c8eec 2023-09-29T06:15:49,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/24a500531dc1459682e20f9ca9489076 is 50, key is test_row_0/B:col10/1695968148529/Put/seqid=0 2023-09-29T06:15:49,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=184 2023-09-29T06:15:49,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:49,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968209478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:49,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:49,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37058 deadline: 1695968209591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:49,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:49,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37034 deadline: 1695968209592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:49,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:49,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37086 deadline: 1695968209595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:49,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:49,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37074 deadline: 1695968209596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:49,771 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/24a500531dc1459682e20f9ca9489076 2023-09-29T06:15:49,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/d6cfc457cb0e45ccb946f96971be1f78 is 50, key is test_row_0/C:col10/1695968148529/Put/seqid=0 2023-09-29T06:15:49,780 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/d6cfc457cb0e45ccb946f96971be1f78 2023-09-29T06:15:49,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/a11a931452fb4e4a993c01a9844c8eec as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/a11a931452fb4e4a993c01a9844c8eec 2023-09-29T06:15:49,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:49,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968209782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:49,788 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/a11a931452fb4e4a993c01a9844c8eec, entries=150, sequenceid=359, filesize=30.5 K 2023-09-29T06:15:49,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/24a500531dc1459682e20f9ca9489076 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/24a500531dc1459682e20f9ca9489076 2023-09-29T06:15:49,792 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/24a500531dc1459682e20f9ca9489076, entries=150, sequenceid=359, filesize=12.0 K 2023-09-29T06:15:49,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/d6cfc457cb0e45ccb946f96971be1f78 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/d6cfc457cb0e45ccb946f96971be1f78 2023-09-29T06:15:49,796 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/d6cfc457cb0e45ccb946f96971be1f78, entries=150, sequenceid=359, filesize=12.0 K 2023-09-29T06:15:49,797 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HRegion(3022): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 0f1c71e3f887ce47e67c17dad51fd610 in 858ms, sequenceid=359, compaction requested=true 2023-09-29T06:15:49,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HRegion(2520): Flush status journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:49,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:49,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=185 2023-09-29T06:15:49,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=185 2023-09-29T06:15:49,799 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=185, resume processing ppid=184 2023-09-29T06:15:49,799 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=185, ppid=184, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4690 sec 2023-09-29T06:15:49,800 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=184, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=184, table=TestAcidGuarantees in 1.4730 sec 2023-09-29T06:15:50,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:50,292 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 0f1c71e3f887ce47e67c17dad51fd610 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2023-09-29T06:15:50,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=A 2023-09-29T06:15:50,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:50,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=B 2023-09-29T06:15:50,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:50,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=C 2023-09-29T06:15:50,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:50,297 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023092924f37dd6a0ab49d0b0a27ef3a2510f7f_0f1c71e3f887ce47e67c17dad51fd610 is 50, key is test_row_0/A:col10/1695968149172/Put/seqid=0 2023-09-29T06:15:50,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:50,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968210356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:50,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=184 2023-09-29T06:15:50,432 INFO [Thread-3332 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 184 completed 2023-09-29T06:15:50,434 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:15:50,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=186, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=186, table=TestAcidGuarantees 2023-09-29T06:15:50,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=186 2023-09-29T06:15:50,436 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=186, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=186, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:15:50,436 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=186, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=186, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:15:50,437 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=187, ppid=186, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:15:50,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:50,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968210459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:50,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=186 2023-09-29T06:15:50,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=187 2023-09-29T06:15:50,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:50,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. as already flushing 2023-09-29T06:15:50,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:50,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] handler.RSProcedureHandler(53): pid=187 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:50,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=187 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:50,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=187 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:50,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:50,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968210661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:50,704 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023092924f37dd6a0ab49d0b0a27ef3a2510f7f_0f1c71e3f887ce47e67c17dad51fd610 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023092924f37dd6a0ab49d0b0a27ef3a2510f7f_0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:50,705 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/396d3914646e47ee8e75015e5865b256, store: [table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:50,705 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/396d3914646e47ee8e75015e5865b256 is 175, key is test_row_0/A:col10/1695968149172/Put/seqid=0 2023-09-29T06:15:50,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=186 2023-09-29T06:15:50,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=187 2023-09-29T06:15:50,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:50,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. as already flushing 2023-09-29T06:15:50,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:50,742 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] handler.RSProcedureHandler(53): pid=187 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:50,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=187 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:50,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=187 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:50,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=187 2023-09-29T06:15:50,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:50,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. as already flushing 2023-09-29T06:15:50,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:50,894 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] handler.RSProcedureHandler(53): pid=187 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:50,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=187 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:50,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=187 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:50,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:50,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968210965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:51,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=186 2023-09-29T06:15:51,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=187 2023-09-29T06:15:51,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:51,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. as already flushing 2023-09-29T06:15:51,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:51,046 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] handler.RSProcedureHandler(53): pid=187 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:51,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=187 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:51,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=187 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:51,118 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=371, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/396d3914646e47ee8e75015e5865b256 2023-09-29T06:15:51,124 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/39962d56e21d48a283d62779b8a55d4c is 50, key is test_row_0/B:col10/1695968149172/Put/seqid=0 2023-09-29T06:15:51,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=187 2023-09-29T06:15:51,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:51,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. as already flushing 2023-09-29T06:15:51,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:51,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] handler.RSProcedureHandler(53): pid=187 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:51,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=187 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:51,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=187 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:51,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=187 2023-09-29T06:15:51,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:51,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. as already flushing 2023-09-29T06:15:51,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:51,351 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] handler.RSProcedureHandler(53): pid=187 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:51,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=187 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:51,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=187 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:51,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:51,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968211469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:51,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=187 2023-09-29T06:15:51,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:51,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. as already flushing 2023-09-29T06:15:51,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:51,504 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] handler.RSProcedureHandler(53): pid=187 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:51,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=187 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:51,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=187 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:51,528 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/39962d56e21d48a283d62779b8a55d4c 2023-09-29T06:15:51,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/4c0702689bb74a41bf4ad2190443bdf5 is 50, key is test_row_0/C:col10/1695968149172/Put/seqid=0 2023-09-29T06:15:51,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=186 2023-09-29T06:15:51,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:51,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37058 deadline: 1695968211596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:51,599 DEBUG [Thread-3324 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4129 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_0' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610., hostname=jenkins-hbase3.apache.org,39783,1695967978138, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:572) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:559) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:197) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:218) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:750) 2023-09-29T06:15:51,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:51,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37074 deadline: 1695968211600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:51,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:51,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37034 deadline: 1695968211600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:51,602 DEBUG [Thread-3326 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4129 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_2' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610., hostname=jenkins-hbase3.apache.org,39783,1695967978138, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:572) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:559) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:197) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:218) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:750) 2023-09-29T06:15:51,604 DEBUG [Thread-3328 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4130 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_0' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610., hostname=jenkins-hbase3.apache.org,39783,1695967978138, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:572) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:559) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:197) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:218) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:750) 2023-09-29T06:15:51,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:51,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37086 deadline: 1695968211609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:51,612 DEBUG [Thread-3330 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4140 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_0' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610., hostname=jenkins-hbase3.apache.org,39783,1695967978138, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:572) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:559) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:197) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:218) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:750) 2023-09-29T06:15:51,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=187 2023-09-29T06:15:51,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:51,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. as already flushing 2023-09-29T06:15:51,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:51,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] handler.RSProcedureHandler(53): pid=187 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:51,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=187 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:51,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=187 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:51,808 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=187 2023-09-29T06:15:51,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:51,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. as already flushing 2023-09-29T06:15:51,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:51,808 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] handler.RSProcedureHandler(53): pid=187 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:51,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=187 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:51,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=187 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:51,937 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/4c0702689bb74a41bf4ad2190443bdf5 2023-09-29T06:15:51,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/396d3914646e47ee8e75015e5865b256 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/396d3914646e47ee8e75015e5865b256 2023-09-29T06:15:51,944 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/396d3914646e47ee8e75015e5865b256, entries=150, sequenceid=371, filesize=30.5 K 2023-09-29T06:15:51,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/39962d56e21d48a283d62779b8a55d4c as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/39962d56e21d48a283d62779b8a55d4c 2023-09-29T06:15:51,948 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/39962d56e21d48a283d62779b8a55d4c, entries=150, sequenceid=371, filesize=12.0 K 2023-09-29T06:15:51,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/4c0702689bb74a41bf4ad2190443bdf5 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/4c0702689bb74a41bf4ad2190443bdf5 2023-09-29T06:15:51,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/4c0702689bb74a41bf4ad2190443bdf5, entries=150, sequenceid=371, filesize=12.0 K 2023-09-29T06:15:51,952 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 0f1c71e3f887ce47e67c17dad51fd610 in 1660ms, sequenceid=371, compaction requested=true 2023-09-29T06:15:51,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:51,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f1c71e3f887ce47e67c17dad51fd610:A, priority=-2147483648, current under compaction store size is 1 2023-09-29T06:15:51,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:51,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f1c71e3f887ce47e67c17dad51fd610:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:15:51,952 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-29T06:15:51,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:51,952 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-29T06:15:51,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f1c71e3f887ce47e67c17dad51fd610:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:15:51,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:15:51,953 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 169208 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-29T06:15:51,953 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 0f1c71e3f887ce47e67c17dad51fd610/A is initiating minor compaction (all files) 2023-09-29T06:15:51,953 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-29T06:15:51,953 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 0f1c71e3f887ce47e67c17dad51fd610/A in TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:51,953 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 0f1c71e3f887ce47e67c17dad51fd610/B is initiating minor compaction (all files) 2023-09-29T06:15:51,953 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/37a13dccf6334906a4b33a3a47fcf33e, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/b9144deae82e4bf9a5c01407faeb0c7c, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/a11a931452fb4e4a993c01a9844c8eec, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/396d3914646e47ee8e75015e5865b256] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp, totalSize=165.2 K 2023-09-29T06:15:51,954 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 0f1c71e3f887ce47e67c17dad51fd610/B in TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:51,954 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:51,954 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/6d6326409315494fa12fdca35314d343, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/913626044c0b4676992d400b1e0e1c41, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/24a500531dc1459682e20f9ca9489076, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/39962d56e21d48a283d62779b8a55d4c] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp, totalSize=48.7 K 2023-09-29T06:15:51,954 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. files: [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/37a13dccf6334906a4b33a3a47fcf33e, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/b9144deae82e4bf9a5c01407faeb0c7c, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/a11a931452fb4e4a993c01a9844c8eec, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/396d3914646e47ee8e75015e5865b256] 2023-09-29T06:15:51,954 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 6d6326409315494fa12fdca35314d343, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1695968147470 2023-09-29T06:15:51,954 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 37a13dccf6334906a4b33a3a47fcf33e, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1695968147470 2023-09-29T06:15:51,955 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 913626044c0b4676992d400b1e0e1c41, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1695968148465 2023-09-29T06:15:51,955 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting b9144deae82e4bf9a5c01407faeb0c7c, keycount=400, bloomtype=ROW, size=73.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1695968147847 2023-09-29T06:15:51,955 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 24a500531dc1459682e20f9ca9489076, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1695968148529 2023-09-29T06:15:51,955 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting a11a931452fb4e4a993c01a9844c8eec, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1695968148529 2023-09-29T06:15:51,955 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 39962d56e21d48a283d62779b8a55d4c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1695968149170 2023-09-29T06:15:51,955 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 396d3914646e47ee8e75015e5865b256, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1695968149170 2023-09-29T06:15:51,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=187 2023-09-29T06:15:51,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:51,961 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.HRegion(2819): Flushing 0f1c71e3f887ce47e67c17dad51fd610 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2023-09-29T06:15:51,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=A 2023-09-29T06:15:51,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:51,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=B 2023-09-29T06:15:51,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:51,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=C 2023-09-29T06:15:51,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:51,967 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:51,969 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f1c71e3f887ce47e67c17dad51fd610#B#compaction#623 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:51,969 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/d2bbf0f79bfe4420b2f7873f35c25ce8 is 50, key is test_row_0/B:col10/1695968149172/Put/seqid=0 2023-09-29T06:15:51,971 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20230929eeb073a25d9b457cb67d9f804295df61_0f1c71e3f887ce47e67c17dad51fd610 store=[table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:51,973 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20230929eeb073a25d9b457cb67d9f804295df61_0f1c71e3f887ce47e67c17dad51fd610, store=[table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:51,974 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929eeb073a25d9b457cb67d9f804295df61_0f1c71e3f887ce47e67c17dad51fd610 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:51,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309290688eadd0f1a450781edba7ad4533531_0f1c71e3f887ce47e67c17dad51fd610 is 50, key is test_row_0/A:col10/1695968150355/Put/seqid=0 2023-09-29T06:15:51,981 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/d2bbf0f79bfe4420b2f7873f35c25ce8 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/d2bbf0f79bfe4420b2f7873f35c25ce8 2023-09-29T06:15:51,985 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 0f1c71e3f887ce47e67c17dad51fd610/B of 0f1c71e3f887ce47e67c17dad51fd610 into d2bbf0f79bfe4420b2f7873f35c25ce8(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:51,985 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:51,985 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610., storeName=0f1c71e3f887ce47e67c17dad51fd610/B, priority=12, startTime=1695968151952; duration=0sec 2023-09-29T06:15:51,985 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:15:51,985 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f1c71e3f887ce47e67c17dad51fd610:B 2023-09-29T06:15:51,985 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2023-09-29T06:15:51,986 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2023-09-29T06:15:51,986 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 0f1c71e3f887ce47e67c17dad51fd610/C is initiating minor compaction (all files) 2023-09-29T06:15:51,986 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 0f1c71e3f887ce47e67c17dad51fd610/C in TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:51,986 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/2726e80b875c4209a4d12a97a1dbbc90, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/b6df0ef94a144f268e79f1b1586a09cd, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/d6cfc457cb0e45ccb946f96971be1f78, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/4c0702689bb74a41bf4ad2190443bdf5] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp, totalSize=48.7 K 2023-09-29T06:15:51,986 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 2726e80b875c4209a4d12a97a1dbbc90, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1695968147470 2023-09-29T06:15:51,987 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting b6df0ef94a144f268e79f1b1586a09cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1695968148465 2023-09-29T06:15:51,987 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting d6cfc457cb0e45ccb946f96971be1f78, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1695968148529 2023-09-29T06:15:51,987 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 4c0702689bb74a41bf4ad2190443bdf5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1695968149170 2023-09-29T06:15:51,993 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f1c71e3f887ce47e67c17dad51fd610#C#compaction#626 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:51,993 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/7ce3c32c82704ae785c089450cff05b7 is 50, key is test_row_0/C:col10/1695968149172/Put/seqid=0 2023-09-29T06:15:52,380 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f1c71e3f887ce47e67c17dad51fd610#A#compaction#624 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:52,381 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/066be9bd9396442e8330c84cb48b6077 is 175, key is test_row_0/A:col10/1695968149172/Put/seqid=0 2023-09-29T06:15:52,384 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309290688eadd0f1a450781edba7ad4533531_0f1c71e3f887ce47e67c17dad51fd610 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309290688eadd0f1a450781edba7ad4533531_0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:52,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/51ab26f526764f0f806ba8d06fff2570, store: [table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:52,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/51ab26f526764f0f806ba8d06fff2570 is 175, key is test_row_0/A:col10/1695968150355/Put/seqid=0 2023-09-29T06:15:52,389 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=395, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/51ab26f526764f0f806ba8d06fff2570 2023-09-29T06:15:52,389 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/066be9bd9396442e8330c84cb48b6077 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/066be9bd9396442e8330c84cb48b6077 2023-09-29T06:15:52,394 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 0f1c71e3f887ce47e67c17dad51fd610/A of 0f1c71e3f887ce47e67c17dad51fd610 into 066be9bd9396442e8330c84cb48b6077(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:52,394 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:52,394 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610., storeName=0f1c71e3f887ce47e67c17dad51fd610/A, priority=12, startTime=1695968151952; duration=0sec 2023-09-29T06:15:52,394 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:52,394 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f1c71e3f887ce47e67c17dad51fd610:A 2023-09-29T06:15:52,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/2312631cad704f38bee91b890f7b81a0 is 50, key is test_row_0/B:col10/1695968150355/Put/seqid=0 2023-09-29T06:15:52,399 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/2312631cad704f38bee91b890f7b81a0 2023-09-29T06:15:52,402 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/7ce3c32c82704ae785c089450cff05b7 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/7ce3c32c82704ae785c089450cff05b7 2023-09-29T06:15:52,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/caee7110a4c74bf8bd5138fe45167f37 is 50, key is test_row_0/C:col10/1695968150355/Put/seqid=0 2023-09-29T06:15:52,407 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 4 (all) file(s) in 0f1c71e3f887ce47e67c17dad51fd610/C of 0f1c71e3f887ce47e67c17dad51fd610 into 7ce3c32c82704ae785c089450cff05b7(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:52,407 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:52,407 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610., storeName=0f1c71e3f887ce47e67c17dad51fd610/C, priority=12, startTime=1695968151952; duration=0sec 2023-09-29T06:15:52,407 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:52,407 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f1c71e3f887ce47e67c17dad51fd610:C 2023-09-29T06:15:52,409 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/caee7110a4c74bf8bd5138fe45167f37 2023-09-29T06:15:52,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/51ab26f526764f0f806ba8d06fff2570 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/51ab26f526764f0f806ba8d06fff2570 2023-09-29T06:15:52,416 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/51ab26f526764f0f806ba8d06fff2570, entries=150, sequenceid=395, filesize=30.5 K 2023-09-29T06:15:52,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/2312631cad704f38bee91b890f7b81a0 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/2312631cad704f38bee91b890f7b81a0 2023-09-29T06:15:52,420 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/2312631cad704f38bee91b890f7b81a0, entries=150, sequenceid=395, filesize=12.0 K 2023-09-29T06:15:52,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/caee7110a4c74bf8bd5138fe45167f37 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/caee7110a4c74bf8bd5138fe45167f37 2023-09-29T06:15:52,424 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/caee7110a4c74bf8bd5138fe45167f37, entries=150, sequenceid=395, filesize=12.0 K 2023-09-29T06:15:52,424 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.HRegion(3022): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=0 B/0 for 0f1c71e3f887ce47e67c17dad51fd610 in 463ms, sequenceid=395, compaction requested=false 2023-09-29T06:15:52,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.HRegion(2520): Flush status journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:52,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:52,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=187 2023-09-29T06:15:52,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=187 2023-09-29T06:15:52,427 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=187, resume processing ppid=186 2023-09-29T06:15:52,427 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=187, ppid=186, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9880 sec 2023-09-29T06:15:52,428 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=186, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=186, table=TestAcidGuarantees in 1.9930 sec 2023-09-29T06:15:52,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:52,534 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 0f1c71e3f887ce47e67c17dad51fd610 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2023-09-29T06:15:52,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=A 2023-09-29T06:15:52,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:52,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=B 2023-09-29T06:15:52,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:52,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=C 2023-09-29T06:15:52,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:52,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=186 2023-09-29T06:15:52,540 INFO [Thread-3332 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 186 completed 2023-09-29T06:15:52,541 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:15:52,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=188, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=188, table=TestAcidGuarantees 2023-09-29T06:15:52,544 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=188, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=188, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:15:52,544 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=188, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=188, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:15:52,544 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=189, ppid=188, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:15:52,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=188 2023-09-29T06:15:52,551 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929422894af6d12440aaa13aabc4fd90d3c_0f1c71e3f887ce47e67c17dad51fd610 is 50, key is test_row_0/A:col10/1695968152533/Put/seqid=0 2023-09-29T06:15:52,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:52,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968212611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:52,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=188 2023-09-29T06:15:52,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=189 2023-09-29T06:15:52,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:52,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. as already flushing 2023-09-29T06:15:52,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:52,698 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=189}] handler.RSProcedureHandler(53): pid=189 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:52,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=189 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:52,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=189 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:52,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:52,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968212714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:52,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=188 2023-09-29T06:15:52,850 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=189 2023-09-29T06:15:52,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:52,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. as already flushing 2023-09-29T06:15:52,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:52,851 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=189}] handler.RSProcedureHandler(53): pid=189 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:52,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=189 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:52,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=189 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:52,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:52,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968212918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:52,959 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929422894af6d12440aaa13aabc4fd90d3c_0f1c71e3f887ce47e67c17dad51fd610 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929422894af6d12440aaa13aabc4fd90d3c_0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:52,959 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/48754350c6814b558e94e6ebd55ba15a, store: [table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:52,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/48754350c6814b558e94e6ebd55ba15a is 175, key is test_row_0/A:col10/1695968152533/Put/seqid=0 2023-09-29T06:15:52,964 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=409, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/48754350c6814b558e94e6ebd55ba15a 2023-09-29T06:15:52,970 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/ce9d9b2624fb41819b5d7e264266d358 is 50, key is test_row_0/B:col10/1695968152533/Put/seqid=0 2023-09-29T06:15:52,973 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/ce9d9b2624fb41819b5d7e264266d358 2023-09-29T06:15:52,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/ecb46689dbdf405bb728219b1fe7b6b0 is 50, key is test_row_0/C:col10/1695968152533/Put/seqid=0 2023-09-29T06:15:52,982 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/ecb46689dbdf405bb728219b1fe7b6b0 2023-09-29T06:15:52,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/48754350c6814b558e94e6ebd55ba15a as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/48754350c6814b558e94e6ebd55ba15a 2023-09-29T06:15:52,989 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/48754350c6814b558e94e6ebd55ba15a, entries=350, sequenceid=409, filesize=64.5 K 2023-09-29T06:15:52,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/ce9d9b2624fb41819b5d7e264266d358 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/ce9d9b2624fb41819b5d7e264266d358 2023-09-29T06:15:52,993 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/ce9d9b2624fb41819b5d7e264266d358, entries=100, sequenceid=409, filesize=9.6 K 2023-09-29T06:15:52,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/ecb46689dbdf405bb728219b1fe7b6b0 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/ecb46689dbdf405bb728219b1fe7b6b0 2023-09-29T06:15:52,996 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/ecb46689dbdf405bb728219b1fe7b6b0, entries=100, sequenceid=409, filesize=9.6 K 2023-09-29T06:15:52,997 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 0f1c71e3f887ce47e67c17dad51fd610 in 463ms, sequenceid=409, compaction requested=true 2023-09-29T06:15:52,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:52,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f1c71e3f887ce47e67c17dad51fd610:A, priority=-2147483648, current under compaction store size is 1 2023-09-29T06:15:52,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:52,997 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:15:52,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f1c71e3f887ce47e67c17dad51fd610:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:15:52,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:52,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f1c71e3f887ce47e67c17dad51fd610:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:15:52,997 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:15:52,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:15:52,998 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 129330 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:15:52,998 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 0f1c71e3f887ce47e67c17dad51fd610/A is initiating minor compaction (all files) 2023-09-29T06:15:52,998 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35260 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:15:52,998 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 0f1c71e3f887ce47e67c17dad51fd610/A in TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:52,998 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 0f1c71e3f887ce47e67c17dad51fd610/B is initiating minor compaction (all files) 2023-09-29T06:15:52,998 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 0f1c71e3f887ce47e67c17dad51fd610/B in TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:52,998 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/066be9bd9396442e8330c84cb48b6077, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/51ab26f526764f0f806ba8d06fff2570, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/48754350c6814b558e94e6ebd55ba15a] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp, totalSize=126.3 K 2023-09-29T06:15:52,998 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/d2bbf0f79bfe4420b2f7873f35c25ce8, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/2312631cad704f38bee91b890f7b81a0, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/ce9d9b2624fb41819b5d7e264266d358] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp, totalSize=34.4 K 2023-09-29T06:15:52,998 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:52,998 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. files: [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/066be9bd9396442e8330c84cb48b6077, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/51ab26f526764f0f806ba8d06fff2570, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/48754350c6814b558e94e6ebd55ba15a] 2023-09-29T06:15:52,999 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting d2bbf0f79bfe4420b2f7873f35c25ce8, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1695968149170 2023-09-29T06:15:52,999 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 066be9bd9396442e8330c84cb48b6077, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1695968149170 2023-09-29T06:15:52,999 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 2312631cad704f38bee91b890f7b81a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1695968150344 2023-09-29T06:15:52,999 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 51ab26f526764f0f806ba8d06fff2570, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1695968150344 2023-09-29T06:15:53,000 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting ce9d9b2624fb41819b5d7e264266d358, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1695968152527 2023-09-29T06:15:53,000 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 48754350c6814b558e94e6ebd55ba15a, keycount=350, bloomtype=ROW, size=64.5 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1695968152501 2023-09-29T06:15:53,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=189 2023-09-29T06:15:53,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:53,003 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.HRegion(2819): Flushing 0f1c71e3f887ce47e67c17dad51fd610 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2023-09-29T06:15:53,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=A 2023-09-29T06:15:53,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:53,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=B 2023-09-29T06:15:53,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:53,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=C 2023-09-29T06:15:53,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:53,026 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:53,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=189}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309296325a3368b164108ae3672fc137ee5c4_0f1c71e3f887ce47e67c17dad51fd610 is 50, key is test_row_0/A:col10/1695968152605/Put/seqid=0 2023-09-29T06:15:53,027 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f1c71e3f887ce47e67c17dad51fd610#B#compaction#632 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:53,027 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/f965076f91bf469fa3e4f3d3fd86c2ed is 50, key is test_row_0/B:col10/1695968152533/Put/seqid=0 2023-09-29T06:15:53,030 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202309293ab7365d262d4c3c95b842ea03ea7844_0f1c71e3f887ce47e67c17dad51fd610 store=[table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:53,031 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202309293ab7365d262d4c3c95b842ea03ea7844_0f1c71e3f887ce47e67c17dad51fd610, store=[table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:53,031 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309293ab7365d262d4c3c95b842ea03ea7844_0f1c71e3f887ce47e67c17dad51fd610 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:53,037 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f1c71e3f887ce47e67c17dad51fd610#A#compaction#633 average throughput is 2.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:53,037 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/a9830e5929114a4597ed597c609560af is 175, key is test_row_0/A:col10/1695968152533/Put/seqid=0 2023-09-29T06:15:53,040 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/f965076f91bf469fa3e4f3d3fd86c2ed as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/f965076f91bf469fa3e4f3d3fd86c2ed 2023-09-29T06:15:53,044 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 0f1c71e3f887ce47e67c17dad51fd610/B of 0f1c71e3f887ce47e67c17dad51fd610 into f965076f91bf469fa3e4f3d3fd86c2ed(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:53,044 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:53,045 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610., storeName=0f1c71e3f887ce47e67c17dad51fd610/B, priority=13, startTime=1695968152997; duration=0sec 2023-09-29T06:15:53,045 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:15:53,045 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f1c71e3f887ce47e67c17dad51fd610:B 2023-09-29T06:15:53,045 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:15:53,045 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/a9830e5929114a4597ed597c609560af as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/a9830e5929114a4597ed597c609560af 2023-09-29T06:15:53,046 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35260 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:15:53,046 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 0f1c71e3f887ce47e67c17dad51fd610/C is initiating minor compaction (all files) 2023-09-29T06:15:53,046 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 0f1c71e3f887ce47e67c17dad51fd610/C in TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:53,046 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/7ce3c32c82704ae785c089450cff05b7, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/caee7110a4c74bf8bd5138fe45167f37, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/ecb46689dbdf405bb728219b1fe7b6b0] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp, totalSize=34.4 K 2023-09-29T06:15:53,046 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 7ce3c32c82704ae785c089450cff05b7, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1695968149170 2023-09-29T06:15:53,047 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting caee7110a4c74bf8bd5138fe45167f37, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1695968150344 2023-09-29T06:15:53,047 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting ecb46689dbdf405bb728219b1fe7b6b0, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1695968152527 2023-09-29T06:15:53,048 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 0f1c71e3f887ce47e67c17dad51fd610/A of 0f1c71e3f887ce47e67c17dad51fd610 into a9830e5929114a4597ed597c609560af(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:53,048 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:53,048 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610., storeName=0f1c71e3f887ce47e67c17dad51fd610/A, priority=13, startTime=1695968152997; duration=0sec 2023-09-29T06:15:53,049 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:53,049 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f1c71e3f887ce47e67c17dad51fd610:A 2023-09-29T06:15:53,052 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f1c71e3f887ce47e67c17dad51fd610#C#compaction#635 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:53,052 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/f7ac435bb0fa49138d2d1e75e80d2f4e is 50, key is test_row_0/C:col10/1695968152533/Put/seqid=0 2023-09-29T06:15:53,059 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/f7ac435bb0fa49138d2d1e75e80d2f4e as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/f7ac435bb0fa49138d2d1e75e80d2f4e 2023-09-29T06:15:53,063 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 0f1c71e3f887ce47e67c17dad51fd610/C of 0f1c71e3f887ce47e67c17dad51fd610 into f7ac435bb0fa49138d2d1e75e80d2f4e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:53,063 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:53,063 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610., storeName=0f1c71e3f887ce47e67c17dad51fd610/C, priority=13, startTime=1695968152997; duration=0sec 2023-09-29T06:15:53,063 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:53,063 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f1c71e3f887ce47e67c17dad51fd610:C 2023-09-29T06:15:53,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=188 2023-09-29T06:15:53,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. as already flushing 2023-09-29T06:15:53,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:53,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:53,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968213236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:53,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:53,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 259 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968213339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:53,437 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309296325a3368b164108ae3672fc137ee5c4_0f1c71e3f887ce47e67c17dad51fd610 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309296325a3368b164108ae3672fc137ee5c4_0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:53,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=189}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/bee1c85ccbc842f39896798d3b86e24e, store: [table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:53,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=189}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/bee1c85ccbc842f39896798d3b86e24e is 175, key is test_row_0/A:col10/1695968152605/Put/seqid=0 2023-09-29T06:15:53,447 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=189}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=434, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/bee1c85ccbc842f39896798d3b86e24e 2023-09-29T06:15:53,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=189}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/04455329bb7f4d16a27b3188ce6a2a7c is 50, key is test_row_0/B:col10/1695968152605/Put/seqid=0 2023-09-29T06:15:53,463 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/04455329bb7f4d16a27b3188ce6a2a7c 2023-09-29T06:15:53,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=189}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/f912d9216a61451c86e999c68a6225d9 is 50, key is test_row_0/C:col10/1695968152605/Put/seqid=0 2023-09-29T06:15:53,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:53,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 261 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968213540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:53,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=188 2023-09-29T06:15:53,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:53,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 263 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968213845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:53,875 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/f912d9216a61451c86e999c68a6225d9 2023-09-29T06:15:53,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/bee1c85ccbc842f39896798d3b86e24e as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/bee1c85ccbc842f39896798d3b86e24e 2023-09-29T06:15:53,882 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/bee1c85ccbc842f39896798d3b86e24e, entries=150, sequenceid=434, filesize=30.5 K 2023-09-29T06:15:53,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/04455329bb7f4d16a27b3188ce6a2a7c as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/04455329bb7f4d16a27b3188ce6a2a7c 2023-09-29T06:15:53,886 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/04455329bb7f4d16a27b3188ce6a2a7c, entries=150, sequenceid=434, filesize=12.0 K 2023-09-29T06:15:53,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/f912d9216a61451c86e999c68a6225d9 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/f912d9216a61451c86e999c68a6225d9 2023-09-29T06:15:53,890 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/f912d9216a61451c86e999c68a6225d9, entries=150, sequenceid=434, filesize=12.0 K 2023-09-29T06:15:53,890 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.HRegion(3022): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 0f1c71e3f887ce47e67c17dad51fd610 in 887ms, sequenceid=434, compaction requested=false 2023-09-29T06:15:53,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.HRegion(2520): Flush status journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:53,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:53,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=189 2023-09-29T06:15:53,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=189 2023-09-29T06:15:53,892 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=189, resume processing ppid=188 2023-09-29T06:15:53,892 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=189, ppid=188, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3470 sec 2023-09-29T06:15:53,894 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=188, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=188, table=TestAcidGuarantees in 1.3520 sec 2023-09-29T06:15:54,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:54,351 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 0f1c71e3f887ce47e67c17dad51fd610 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2023-09-29T06:15:54,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=A 2023-09-29T06:15:54,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:54,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=B 2023-09-29T06:15:54,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:54,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=C 2023-09-29T06:15:54,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:54,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929f12a498e4b61444185bb8e9711523086_0f1c71e3f887ce47e67c17dad51fd610 is 50, key is test_row_0/A:col10/1695968153233/Put/seqid=0 2023-09-29T06:15:54,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:54,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 287 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968214399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:54,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:54,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 289 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968214501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:54,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=188 2023-09-29T06:15:54,651 INFO [Thread-3332 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 188 completed 2023-09-29T06:15:54,652 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:15:54,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=190, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=190, table=TestAcidGuarantees 2023-09-29T06:15:54,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=190 2023-09-29T06:15:54,654 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=190, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=190, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:15:54,655 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=190, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=190, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:15:54,655 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=191, ppid=190, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:15:54,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:54,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 291 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968214705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:54,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=190 2023-09-29T06:15:54,762 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929f12a498e4b61444185bb8e9711523086_0f1c71e3f887ce47e67c17dad51fd610 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929f12a498e4b61444185bb8e9711523086_0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:54,763 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/83b97c4249d149ff874c1015454529a3, store: [table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:54,764 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/83b97c4249d149ff874c1015454529a3 is 175, key is test_row_0/A:col10/1695968153233/Put/seqid=0 2023-09-29T06:15:54,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=191 2023-09-29T06:15:54,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:54,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. as already flushing 2023-09-29T06:15:54,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:54,807 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] handler.RSProcedureHandler(53): pid=191 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:54,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=191 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:54,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=191 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:54,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=190 2023-09-29T06:15:54,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=191 2023-09-29T06:15:54,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:54,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. as already flushing 2023-09-29T06:15:54,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:54,960 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=191}] handler.RSProcedureHandler(53): pid=191 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:54,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=191 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:54,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=191 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:55,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:55,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 293 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968215008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:55,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=191 2023-09-29T06:15:55,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:55,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. as already flushing 2023-09-29T06:15:55,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:55,113 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=191}] handler.RSProcedureHandler(53): pid=191 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:55,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=191 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:55,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=191 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:55,167 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=449, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/83b97c4249d149ff874c1015454529a3 2023-09-29T06:15:55,173 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/25650ae36f3341ddade2842bc8e7f899 is 50, key is test_row_0/B:col10/1695968153233/Put/seqid=0 2023-09-29T06:15:55,182 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/25650ae36f3341ddade2842bc8e7f899 2023-09-29T06:15:55,188 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/1676483468344f1c94b442f176a9cf33 is 50, key is test_row_0/C:col10/1695968153233/Put/seqid=0 2023-09-29T06:15:55,207 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/1676483468344f1c94b442f176a9cf33 2023-09-29T06:15:55,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/83b97c4249d149ff874c1015454529a3 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/83b97c4249d149ff874c1015454529a3 2023-09-29T06:15:55,214 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/83b97c4249d149ff874c1015454529a3, entries=200, sequenceid=449, filesize=39.0 K 2023-09-29T06:15:55,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/25650ae36f3341ddade2842bc8e7f899 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/25650ae36f3341ddade2842bc8e7f899 2023-09-29T06:15:55,222 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/25650ae36f3341ddade2842bc8e7f899, entries=150, sequenceid=449, filesize=12.0 K 2023-09-29T06:15:55,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/1676483468344f1c94b442f176a9cf33 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/1676483468344f1c94b442f176a9cf33 2023-09-29T06:15:55,226 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/1676483468344f1c94b442f176a9cf33, entries=150, sequenceid=449, filesize=12.0 K 2023-09-29T06:15:55,226 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 0f1c71e3f887ce47e67c17dad51fd610 in 875ms, sequenceid=449, compaction requested=true 2023-09-29T06:15:55,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:55,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f1c71e3f887ce47e67c17dad51fd610:A, priority=-2147483648, current under compaction store size is 1 2023-09-29T06:15:55,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:55,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f1c71e3f887ce47e67c17dad51fd610:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:15:55,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:55,227 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:15:55,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f1c71e3f887ce47e67c17dad51fd610:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:15:55,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:15:55,227 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:15:55,227 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103469 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:15:55,227 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 0f1c71e3f887ce47e67c17dad51fd610/A is initiating minor compaction (all files) 2023-09-29T06:15:55,228 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 0f1c71e3f887ce47e67c17dad51fd610/A in TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:55,228 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37806 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:15:55,228 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 0f1c71e3f887ce47e67c17dad51fd610/B is initiating minor compaction (all files) 2023-09-29T06:15:55,228 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/a9830e5929114a4597ed597c609560af, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/bee1c85ccbc842f39896798d3b86e24e, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/83b97c4249d149ff874c1015454529a3] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp, totalSize=101.0 K 2023-09-29T06:15:55,228 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 0f1c71e3f887ce47e67c17dad51fd610/B in TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:55,228 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:55,228 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/f965076f91bf469fa3e4f3d3fd86c2ed, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/04455329bb7f4d16a27b3188ce6a2a7c, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/25650ae36f3341ddade2842bc8e7f899] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp, totalSize=36.9 K 2023-09-29T06:15:55,228 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. files: [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/a9830e5929114a4597ed597c609560af, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/bee1c85ccbc842f39896798d3b86e24e, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/83b97c4249d149ff874c1015454529a3] 2023-09-29T06:15:55,228 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting f965076f91bf469fa3e4f3d3fd86c2ed, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1695968150344 2023-09-29T06:15:55,228 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting a9830e5929114a4597ed597c609560af, keycount=150, bloomtype=ROW, size=31.5 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1695968150344 2023-09-29T06:15:55,229 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 04455329bb7f4d16a27b3188ce6a2a7c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1695968152605 2023-09-29T06:15:55,229 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting bee1c85ccbc842f39896798d3b86e24e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1695968152605 2023-09-29T06:15:55,229 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 25650ae36f3341ddade2842bc8e7f899, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1695968153233 2023-09-29T06:15:55,229 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 83b97c4249d149ff874c1015454529a3, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1695968153231 2023-09-29T06:15:55,234 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:55,235 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f1c71e3f887ce47e67c17dad51fd610#B#compaction#641 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:55,236 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/b7bbd4104f8f49fab29084e66d49c543 is 50, key is test_row_0/B:col10/1695968153233/Put/seqid=0 2023-09-29T06:15:55,238 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2023092954b5b051976f4ba0b1c886b3063881c2_0f1c71e3f887ce47e67c17dad51fd610 store=[table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:55,239 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2023092954b5b051976f4ba0b1c886b3063881c2_0f1c71e3f887ce47e67c17dad51fd610, store=[table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:55,239 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023092954b5b051976f4ba0b1c886b3063881c2_0f1c71e3f887ce47e67c17dad51fd610 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:55,249 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f1c71e3f887ce47e67c17dad51fd610#A#compaction#642 average throughput is 1.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:55,249 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/4ec3ec9b066b4c1c96528957d72cb685 is 175, key is test_row_0/A:col10/1695968153233/Put/seqid=0 2023-09-29T06:15:55,256 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/4ec3ec9b066b4c1c96528957d72cb685 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/4ec3ec9b066b4c1c96528957d72cb685 2023-09-29T06:15:55,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=190 2023-09-29T06:15:55,260 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 0f1c71e3f887ce47e67c17dad51fd610/A of 0f1c71e3f887ce47e67c17dad51fd610 into 4ec3ec9b066b4c1c96528957d72cb685(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:55,260 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:55,260 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610., storeName=0f1c71e3f887ce47e67c17dad51fd610/A, priority=13, startTime=1695968155226; duration=0sec 2023-09-29T06:15:55,260 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:15:55,260 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f1c71e3f887ce47e67c17dad51fd610:A 2023-09-29T06:15:55,260 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:15:55,261 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37806 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:15:55,261 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 0f1c71e3f887ce47e67c17dad51fd610/C is initiating minor compaction (all files) 2023-09-29T06:15:55,261 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 0f1c71e3f887ce47e67c17dad51fd610/C in TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:55,261 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/f7ac435bb0fa49138d2d1e75e80d2f4e, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/f912d9216a61451c86e999c68a6225d9, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/1676483468344f1c94b442f176a9cf33] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp, totalSize=36.9 K 2023-09-29T06:15:55,261 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting f7ac435bb0fa49138d2d1e75e80d2f4e, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1695968150344 2023-09-29T06:15:55,262 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting f912d9216a61451c86e999c68a6225d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1695968152605 2023-09-29T06:15:55,262 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 1676483468344f1c94b442f176a9cf33, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1695968153233 2023-09-29T06:15:55,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=191 2023-09-29T06:15:55,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:55,265 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.HRegion(2819): Flushing 0f1c71e3f887ce47e67c17dad51fd610 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2023-09-29T06:15:55,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=A 2023-09-29T06:15:55,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:55,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=B 2023-09-29T06:15:55,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:55,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=C 2023-09-29T06:15:55,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:55,268 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f1c71e3f887ce47e67c17dad51fd610#C#compaction#643 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:55,269 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/0140649f8e214161ba2dca673c9f9564 is 50, key is test_row_0/C:col10/1695968153233/Put/seqid=0 2023-09-29T06:15:55,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309297e512fe23c6a4d4792fdcd3cce6e2627_0f1c71e3f887ce47e67c17dad51fd610 is 50, key is test_row_0/A:col10/1695968154371/Put/seqid=0 2023-09-29T06:15:55,287 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309297e512fe23c6a4d4792fdcd3cce6e2627_0f1c71e3f887ce47e67c17dad51fd610 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309297e512fe23c6a4d4792fdcd3cce6e2627_0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:55,288 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/0140649f8e214161ba2dca673c9f9564 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/0140649f8e214161ba2dca673c9f9564 2023-09-29T06:15:55,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/54dafbd31cbb498f8ebcf452691117c4, store: [table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:55,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/54dafbd31cbb498f8ebcf452691117c4 is 175, key is test_row_0/A:col10/1695968154371/Put/seqid=0 2023-09-29T06:15:55,292 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 0f1c71e3f887ce47e67c17dad51fd610/C of 0f1c71e3f887ce47e67c17dad51fd610 into 0140649f8e214161ba2dca673c9f9564(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:55,292 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:55,293 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610., storeName=0f1c71e3f887ce47e67c17dad51fd610/C, priority=13, startTime=1695968155227; duration=0sec 2023-09-29T06:15:55,293 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:55,293 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f1c71e3f887ce47e67c17dad51fd610:C 2023-09-29T06:15:55,293 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=474, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/54dafbd31cbb498f8ebcf452691117c4 2023-09-29T06:15:55,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/939dca46d14d40e1b0b440a842645865 is 50, key is test_row_0/B:col10/1695968154371/Put/seqid=0 2023-09-29T06:15:55,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:55,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. as already flushing 2023-09-29T06:15:55,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:55,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 304 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968215534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:55,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:55,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37086 deadline: 1695968215624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:55,626 DEBUG [Thread-3330 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=7, retries=16, started=8154 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_0' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610., hostname=jenkins-hbase3.apache.org,39783,1695967978138, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:572) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:559) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:197) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:218) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:750) 2023-09-29T06:15:55,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:55,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37074 deadline: 1695968215636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:55,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:55,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37058 deadline: 1695968215636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:55,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:55,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37034 deadline: 1695968215636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:55,638 DEBUG [Thread-3326 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=7, retries=16, started=8165 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_2' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610., hostname=jenkins-hbase3.apache.org,39783,1695967978138, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:572) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:559) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:197) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:218) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:750) 2023-09-29T06:15:55,639 DEBUG [Thread-3324 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=7, retries=16, started=8169 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_0' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610., hostname=jenkins-hbase3.apache.org,39783,1695967978138, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:572) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:559) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:197) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:218) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:750) 2023-09-29T06:15:55,639 DEBUG [Thread-3328 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=7, retries=16, started=8166 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_0' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610., hostname=jenkins-hbase3.apache.org,39783,1695967978138, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:572) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:559) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:197) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:218) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:750) 2023-09-29T06:15:55,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:55,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 306 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968215637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:55,652 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/b7bbd4104f8f49fab29084e66d49c543 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/b7bbd4104f8f49fab29084e66d49c543 2023-09-29T06:15:55,655 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 0f1c71e3f887ce47e67c17dad51fd610/B of 0f1c71e3f887ce47e67c17dad51fd610 into b7bbd4104f8f49fab29084e66d49c543(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:55,655 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:55,655 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610., storeName=0f1c71e3f887ce47e67c17dad51fd610/B, priority=13, startTime=1695968155227; duration=0sec 2023-09-29T06:15:55,655 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:55,655 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f1c71e3f887ce47e67c17dad51fd610:B 2023-09-29T06:15:55,702 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=474 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/939dca46d14d40e1b0b440a842645865 2023-09-29T06:15:55,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/f3fb36313ef04ed6a9d268673cd492b5 is 50, key is test_row_0/C:col10/1695968154371/Put/seqid=0 2023-09-29T06:15:55,711 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=474 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/f3fb36313ef04ed6a9d268673cd492b5 2023-09-29T06:15:55,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/54dafbd31cbb498f8ebcf452691117c4 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/54dafbd31cbb498f8ebcf452691117c4 2023-09-29T06:15:55,717 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/54dafbd31cbb498f8ebcf452691117c4, entries=150, sequenceid=474, filesize=30.5 K 2023-09-29T06:15:55,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/939dca46d14d40e1b0b440a842645865 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/939dca46d14d40e1b0b440a842645865 2023-09-29T06:15:55,721 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/939dca46d14d40e1b0b440a842645865, entries=150, sequenceid=474, filesize=12.0 K 2023-09-29T06:15:55,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/f3fb36313ef04ed6a9d268673cd492b5 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/f3fb36313ef04ed6a9d268673cd492b5 2023-09-29T06:15:55,724 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/f3fb36313ef04ed6a9d268673cd492b5, entries=150, sequenceid=474, filesize=12.0 K 2023-09-29T06:15:55,725 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.HRegion(3022): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 0f1c71e3f887ce47e67c17dad51fd610 in 460ms, sequenceid=474, compaction requested=false 2023-09-29T06:15:55,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.HRegion(2520): Flush status journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:55,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:55,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=191 2023-09-29T06:15:55,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=191 2023-09-29T06:15:55,727 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=191, resume processing ppid=190 2023-09-29T06:15:55,727 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=191, ppid=190, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0710 sec 2023-09-29T06:15:55,728 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=190, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=190, table=TestAcidGuarantees in 1.0750 sec 2023-09-29T06:15:55,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=190 2023-09-29T06:15:55,757 INFO [Thread-3332 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 190 completed 2023-09-29T06:15:55,759 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster$21(4274): Client=jenkins//172.31.12.81 flush TestAcidGuarantees 2023-09-29T06:15:55,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=192, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=192, table=TestAcidGuarantees 2023-09-29T06:15:55,761 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=192, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=192, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2023-09-29T06:15:55,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=192 2023-09-29T06:15:55,761 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=192, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=192, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2023-09-29T06:15:55,761 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=193, ppid=192, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2023-09-29T06:15:55,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:55,843 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 0f1c71e3f887ce47e67c17dad51fd610 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2023-09-29T06:15:55,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=A 2023-09-29T06:15:55,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:55,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=B 2023-09-29T06:15:55,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:55,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=C 2023-09-29T06:15:55,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:55,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309293aa3234a0283406fa5ea1d42bf6603a5_0f1c71e3f887ce47e67c17dad51fd610 is 50, key is test_row_0/A:col10/1695968155842/Put/seqid=0 2023-09-29T06:15:55,855 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309293aa3234a0283406fa5ea1d42bf6603a5_0f1c71e3f887ce47e67c17dad51fd610 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309293aa3234a0283406fa5ea1d42bf6603a5_0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:55,856 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/50648b3f04924881b0201f08431112e1, store: [table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:55,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/50648b3f04924881b0201f08431112e1 is 175, key is test_row_0/A:col10/1695968155842/Put/seqid=0 2023-09-29T06:15:55,860 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=489, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/50648b3f04924881b0201f08431112e1 2023-09-29T06:15:55,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=192 2023-09-29T06:15:55,865 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/301f1393a4fe4405aa2344e4386099d3 is 50, key is test_row_0/B:col10/1695968155842/Put/seqid=0 2023-09-29T06:15:55,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:55,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 329 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968215895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:55,914 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=193 2023-09-29T06:15:55,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:55,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. as already flushing 2023-09-29T06:15:55,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:55,915 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=193}] handler.RSProcedureHandler(53): pid=193 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:55,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=193 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:55,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=193 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:55,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:55,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 331 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968215998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:56,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=192 2023-09-29T06:15:56,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=193 2023-09-29T06:15:56,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:56,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. as already flushing 2023-09-29T06:15:56,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:56,067 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=193}] handler.RSProcedureHandler(53): pid=193 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:56,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=193 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:56,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=193 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:56,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:56,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 333 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968216200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:56,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=193 2023-09-29T06:15:56,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:56,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. as already flushing 2023-09-29T06:15:56,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:56,218 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] handler.RSProcedureHandler(53): pid=193 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:56,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=193 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:56,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=193 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:56,278 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/301f1393a4fe4405aa2344e4386099d3 2023-09-29T06:15:56,283 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/8d43366393c84aa4a1b5327599fb2cba is 50, key is test_row_0/C:col10/1695968155842/Put/seqid=0 2023-09-29T06:15:56,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=192 2023-09-29T06:15:56,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=193 2023-09-29T06:15:56,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:56,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. as already flushing 2023-09-29T06:15:56,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:56,371 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=193}] handler.RSProcedureHandler(53): pid=193 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:56,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=193 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:56,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=193 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:56,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:56,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 335 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968216502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:56,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=193 2023-09-29T06:15:56,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:56,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. as already flushing 2023-09-29T06:15:56,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:56,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=193}] handler.RSProcedureHandler(53): pid=193 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:56,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=193 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:56,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=193 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:56,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=193 2023-09-29T06:15:56,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:56,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. as already flushing 2023-09-29T06:15:56,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:56,675 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] handler.RSProcedureHandler(53): pid=193 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:56,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=193 java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:56,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(4003): Remote procedure failed, pid=193 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2540) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17405) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-29T06:15:56,688 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/8d43366393c84aa4a1b5327599fb2cba 2023-09-29T06:15:56,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/50648b3f04924881b0201f08431112e1 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/50648b3f04924881b0201f08431112e1 2023-09-29T06:15:56,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/50648b3f04924881b0201f08431112e1, entries=150, sequenceid=489, filesize=30.5 K 2023-09-29T06:15:56,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/301f1393a4fe4405aa2344e4386099d3 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/301f1393a4fe4405aa2344e4386099d3 2023-09-29T06:15:56,699 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/301f1393a4fe4405aa2344e4386099d3, entries=150, sequenceid=489, filesize=12.0 K 2023-09-29T06:15:56,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/8d43366393c84aa4a1b5327599fb2cba as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/8d43366393c84aa4a1b5327599fb2cba 2023-09-29T06:15:56,703 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/8d43366393c84aa4a1b5327599fb2cba, entries=150, sequenceid=489, filesize=12.0 K 2023-09-29T06:15:56,704 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 0f1c71e3f887ce47e67c17dad51fd610 in 861ms, sequenceid=489, compaction requested=true 2023-09-29T06:15:56,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:56,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f1c71e3f887ce47e67c17dad51fd610:A, priority=-2147483648, current under compaction store size is 1 2023-09-29T06:15:56,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:56,704 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:15:56,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f1c71e3f887ce47e67c17dad51fd610:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:15:56,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:56,704 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:15:56,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f1c71e3f887ce47e67c17dad51fd610:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:15:56,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:15:56,706 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37908 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:15:56,706 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94770 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:15:56,706 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 0f1c71e3f887ce47e67c17dad51fd610/A is initiating minor compaction (all files) 2023-09-29T06:15:56,706 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 0f1c71e3f887ce47e67c17dad51fd610/B is initiating minor compaction (all files) 2023-09-29T06:15:56,706 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 0f1c71e3f887ce47e67c17dad51fd610/A in TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:56,706 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 0f1c71e3f887ce47e67c17dad51fd610/B in TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:56,706 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/4ec3ec9b066b4c1c96528957d72cb685, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/54dafbd31cbb498f8ebcf452691117c4, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/50648b3f04924881b0201f08431112e1] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp, totalSize=92.5 K 2023-09-29T06:15:56,706 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:56,706 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/b7bbd4104f8f49fab29084e66d49c543, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/939dca46d14d40e1b0b440a842645865, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/301f1393a4fe4405aa2344e4386099d3] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp, totalSize=37.0 K 2023-09-29T06:15:56,706 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. files: [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/4ec3ec9b066b4c1c96528957d72cb685, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/54dafbd31cbb498f8ebcf452691117c4, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/50648b3f04924881b0201f08431112e1] 2023-09-29T06:15:56,706 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 4ec3ec9b066b4c1c96528957d72cb685, keycount=150, bloomtype=ROW, size=31.5 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1695968153233 2023-09-29T06:15:56,706 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting b7bbd4104f8f49fab29084e66d49c543, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1695968153233 2023-09-29T06:15:56,707 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 54dafbd31cbb498f8ebcf452691117c4, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1695968154371 2023-09-29T06:15:56,707 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 939dca46d14d40e1b0b440a842645865, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1695968154371 2023-09-29T06:15:56,707 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 50648b3f04924881b0201f08431112e1, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=489, earliestPutTs=1695968155530 2023-09-29T06:15:56,707 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 301f1393a4fe4405aa2344e4386099d3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=489, earliestPutTs=1695968155530 2023-09-29T06:15:56,713 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:56,714 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20230929cd17f3ab52c2424e9e0e3178d93ad6f4_0f1c71e3f887ce47e67c17dad51fd610 store=[table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:56,714 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f1c71e3f887ce47e67c17dad51fd610#B#compaction#651 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:56,715 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/75aec740ea35434b9928b32008e6b1ad is 50, key is test_row_0/B:col10/1695968155842/Put/seqid=0 2023-09-29T06:15:56,715 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20230929cd17f3ab52c2424e9e0e3178d93ad6f4_0f1c71e3f887ce47e67c17dad51fd610, store=[table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:56,716 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929cd17f3ab52c2424e9e0e3178d93ad6f4_0f1c71e3f887ce47e67c17dad51fd610 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:56,720 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f1c71e3f887ce47e67c17dad51fd610#A#compaction#650 average throughput is 3.49 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:56,721 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/24a943df07aa48c6831ede40991666f3 is 175, key is test_row_0/A:col10/1695968155842/Put/seqid=0 2023-09-29T06:15:56,723 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/75aec740ea35434b9928b32008e6b1ad as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/75aec740ea35434b9928b32008e6b1ad 2023-09-29T06:15:56,727 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 0f1c71e3f887ce47e67c17dad51fd610/B of 0f1c71e3f887ce47e67c17dad51fd610 into 75aec740ea35434b9928b32008e6b1ad(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:56,727 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:56,727 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610., storeName=0f1c71e3f887ce47e67c17dad51fd610/B, priority=13, startTime=1695968156704; duration=0sec 2023-09-29T06:15:56,727 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:15:56,727 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f1c71e3f887ce47e67c17dad51fd610:B 2023-09-29T06:15:56,727 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:15:56,728 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37908 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:15:56,728 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 0f1c71e3f887ce47e67c17dad51fd610/C is initiating minor compaction (all files) 2023-09-29T06:15:56,728 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 0f1c71e3f887ce47e67c17dad51fd610/C in TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:56,729 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/0140649f8e214161ba2dca673c9f9564, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/f3fb36313ef04ed6a9d268673cd492b5, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/8d43366393c84aa4a1b5327599fb2cba] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp, totalSize=37.0 K 2023-09-29T06:15:56,729 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 0140649f8e214161ba2dca673c9f9564, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1695968153233 2023-09-29T06:15:56,729 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting f3fb36313ef04ed6a9d268673cd492b5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1695968154371 2023-09-29T06:15:56,729 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/24a943df07aa48c6831ede40991666f3 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/24a943df07aa48c6831ede40991666f3 2023-09-29T06:15:56,730 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 8d43366393c84aa4a1b5327599fb2cba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=489, earliestPutTs=1695968155530 2023-09-29T06:15:56,734 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 0f1c71e3f887ce47e67c17dad51fd610/A of 0f1c71e3f887ce47e67c17dad51fd610 into 24a943df07aa48c6831ede40991666f3(size=31.6 K), total size for store is 31.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:56,734 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:56,735 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610., storeName=0f1c71e3f887ce47e67c17dad51fd610/A, priority=13, startTime=1695968156704; duration=0sec 2023-09-29T06:15:56,735 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:56,735 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f1c71e3f887ce47e67c17dad51fd610:A 2023-09-29T06:15:56,739 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f1c71e3f887ce47e67c17dad51fd610#C#compaction#652 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:56,739 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/1f364328b01b4832aa7f7128083bdcda is 50, key is test_row_0/C:col10/1695968155842/Put/seqid=0 2023-09-29T06:15:56,746 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/1f364328b01b4832aa7f7128083bdcda as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/1f364328b01b4832aa7f7128083bdcda 2023-09-29T06:15:56,750 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 0f1c71e3f887ce47e67c17dad51fd610/C of 0f1c71e3f887ce47e67c17dad51fd610 into 1f364328b01b4832aa7f7128083bdcda(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:56,750 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:56,750 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610., storeName=0f1c71e3f887ce47e67c17dad51fd610/C, priority=13, startTime=1695968156704; duration=0sec 2023-09-29T06:15:56,750 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:56,751 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f1c71e3f887ce47e67c17dad51fd610:C 2023-09-29T06:15:56,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39783 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=193 2023-09-29T06:15:56,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:56,828 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.HRegion(2819): Flushing 0f1c71e3f887ce47e67c17dad51fd610 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2023-09-29T06:15:56,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=A 2023-09-29T06:15:56,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:56,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=B 2023-09-29T06:15:56,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:56,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=C 2023-09-29T06:15:56,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:56,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=193}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309298a74f6092e704332aae6502ae3a1b621_0f1c71e3f887ce47e67c17dad51fd610 is 50, key is test_row_0/A:col10/1695968155889/Put/seqid=0 2023-09-29T06:15:56,840 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309298a74f6092e704332aae6502ae3a1b621_0f1c71e3f887ce47e67c17dad51fd610 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309298a74f6092e704332aae6502ae3a1b621_0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:56,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=193}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/dec21e52bb8742bf9d59bb26bda389ad, store: [table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:56,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=193}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/dec21e52bb8742bf9d59bb26bda389ad is 175, key is test_row_0/A:col10/1695968155889/Put/seqid=0 2023-09-29T06:15:56,845 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=193}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=515, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/dec21e52bb8742bf9d59bb26bda389ad 2023-09-29T06:15:56,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=193}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/40ec9a7f5e074a9a87e606f54cf372d7 is 50, key is test_row_0/B:col10/1695968155889/Put/seqid=0 2023-09-29T06:15:56,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=192 2023-09-29T06:15:57,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:57,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2478): NOT flushing TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. as already flushing 2023-09-29T06:15:57,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:57,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 347 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968217022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:57,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:57,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 349 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968217124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:57,253 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=515 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/40ec9a7f5e074a9a87e606f54cf372d7 2023-09-29T06:15:57,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=193}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/ff12dde1ecee4d7284c1705a5eb72fd6 is 50, key is test_row_0/C:col10/1695968155889/Put/seqid=0 2023-09-29T06:15:57,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:57,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 351 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968217326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:57,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(5041): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5039) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3211) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3204) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3015) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2978) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44994) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-29T06:15:57,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] ipc.CallRunner(138): callId: 353 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.12.81:37026 deadline: 1695968217627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:15:57,662 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=515 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/ff12dde1ecee4d7284c1705a5eb72fd6 2023-09-29T06:15:57,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/dec21e52bb8742bf9d59bb26bda389ad as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/dec21e52bb8742bf9d59bb26bda389ad 2023-09-29T06:15:57,669 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/dec21e52bb8742bf9d59bb26bda389ad, entries=150, sequenceid=515, filesize=30.5 K 2023-09-29T06:15:57,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/40ec9a7f5e074a9a87e606f54cf372d7 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/40ec9a7f5e074a9a87e606f54cf372d7 2023-09-29T06:15:57,673 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/40ec9a7f5e074a9a87e606f54cf372d7, entries=150, sequenceid=515, filesize=12.0 K 2023-09-29T06:15:57,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/ff12dde1ecee4d7284c1705a5eb72fd6 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/ff12dde1ecee4d7284c1705a5eb72fd6 2023-09-29T06:15:57,676 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/ff12dde1ecee4d7284c1705a5eb72fd6, entries=150, sequenceid=515, filesize=12.0 K 2023-09-29T06:15:57,677 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.HRegion(3022): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 0f1c71e3f887ce47e67c17dad51fd610 in 849ms, sequenceid=515, compaction requested=false 2023-09-29T06:15:57,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.HRegion(2520): Flush status journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:57,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:57,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=193 2023-09-29T06:15:57,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster(3995): Remote procedure done, pid=193 2023-09-29T06:15:57,679 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=193, resume processing ppid=192 2023-09-29T06:15:57,679 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=193, ppid=192, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9170 sec 2023-09-29T06:15:57,680 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=192, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=192, table=TestAcidGuarantees in 1.9200 sec 2023-09-29T06:15:57,766 DEBUG [Thread-3339 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x3f4e9059 to 127.0.0.1:49344 2023-09-29T06:15:57,766 DEBUG [Thread-3339 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:15:57,766 DEBUG [Thread-3337 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x29a1b140 to 127.0.0.1:49344 2023-09-29T06:15:57,766 DEBUG [Thread-3337 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:15:57,767 DEBUG [Thread-3333 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x3116ea32 to 127.0.0.1:49344 2023-09-29T06:15:57,767 DEBUG [Thread-3333 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:15:57,768 DEBUG [Thread-3335 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0e0e44d4 to 127.0.0.1:49344 2023-09-29T06:15:57,768 DEBUG [Thread-3341 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x1c03d8f4 to 127.0.0.1:49344 2023-09-29T06:15:57,768 DEBUG [Thread-3341 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:15:57,768 DEBUG [Thread-3335 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:15:57,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=192 2023-09-29T06:15:57,865 INFO [Thread-3332 {}] client.HBaseAdmin$TableFuture(3677): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 192 completed 2023-09-29T06:15:58,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39783 {}] regionserver.HRegion(8537): Flush requested on 0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:58,134 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2819): Flushing 0f1c71e3f887ce47e67c17dad51fd610 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2023-09-29T06:15:58,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=A 2023-09-29T06:15:58,134 DEBUG [Thread-3322 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x51685a58 to 127.0.0.1:49344 2023-09-29T06:15:58,134 DEBUG [Thread-3322 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:15:58,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:58,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=B 2023-09-29T06:15:58,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:58,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=C 2023-09-29T06:15:58,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:15:58,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023092990e5285d1fd249e9a5e5c685f8dc4d9c_0f1c71e3f887ce47e67c17dad51fd610 is 50, key is test_row_0/A:col10/1695968157017/Put/seqid=0 2023-09-29T06:15:58,544 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2023092990e5285d1fd249e9a5e5c685f8dc4d9c_0f1c71e3f887ce47e67c17dad51fd610 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023092990e5285d1fd249e9a5e5c685f8dc4d9c_0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:15:58,545 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/ac8307b719234b129ea74bc12b4b6372, store: [table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:58,545 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/ac8307b719234b129ea74bc12b4b6372 is 175, key is test_row_0/A:col10/1695968157017/Put/seqid=0 2023-09-29T06:15:58,949 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=529, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/ac8307b719234b129ea74bc12b4b6372 2023-09-29T06:15:58,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/7c66252782774860bd09b51b88ac19e9 is 50, key is test_row_0/B:col10/1695968157017/Put/seqid=0 2023-09-29T06:15:59,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=529 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/7c66252782774860bd09b51b88ac19e9 2023-09-29T06:15:59,364 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/c2dc7aea006641f4971b3871f1930104 is 50, key is test_row_0/C:col10/1695968157017/Put/seqid=0 2023-09-29T06:15:59,772 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=529 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/c2dc7aea006641f4971b3871f1930104 2023-09-29T06:15:59,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/ac8307b719234b129ea74bc12b4b6372 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/ac8307b719234b129ea74bc12b4b6372 2023-09-29T06:15:59,778 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/ac8307b719234b129ea74bc12b4b6372, entries=150, sequenceid=529, filesize=30.5 K 2023-09-29T06:15:59,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/7c66252782774860bd09b51b88ac19e9 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/7c66252782774860bd09b51b88ac19e9 2023-09-29T06:15:59,781 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/7c66252782774860bd09b51b88ac19e9, entries=150, sequenceid=529, filesize=12.0 K 2023-09-29T06:15:59,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/c2dc7aea006641f4971b3871f1930104 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/c2dc7aea006641f4971b3871f1930104 2023-09-29T06:15:59,785 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/c2dc7aea006641f4971b3871f1930104, entries=150, sequenceid=529, filesize=12.0 K 2023-09-29T06:15:59,785 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3022): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=0 B/0 for 0f1c71e3f887ce47e67c17dad51fd610 in 1651ms, sequenceid=529, compaction requested=true 2023-09-29T06:15:59,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2520): Flush status journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:59,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f1c71e3f887ce47e67c17dad51fd610:A, priority=-2147483648, current under compaction store size is 1 2023-09-29T06:15:59,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:59,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f1c71e3f887ce47e67c17dad51fd610:B, priority=-2147483648, current under compaction store size is 2 2023-09-29T06:15:59,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:15:59,785 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:15:59,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f1c71e3f887ce47e67c17dad51fd610:C, priority=-2147483648, current under compaction store size is 3 2023-09-29T06:15:59,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:15:59,786 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:15:59,786 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94872 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:15:59,786 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38010 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:15:59,786 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1534): 0f1c71e3f887ce47e67c17dad51fd610/A is initiating minor compaction (all files) 2023-09-29T06:15:59,786 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 0f1c71e3f887ce47e67c17dad51fd610/B is initiating minor compaction (all files) 2023-09-29T06:15:59,786 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 0f1c71e3f887ce47e67c17dad51fd610/A in TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:59,786 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 0f1c71e3f887ce47e67c17dad51fd610/B in TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:59,786 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/24a943df07aa48c6831ede40991666f3, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/dec21e52bb8742bf9d59bb26bda389ad, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/ac8307b719234b129ea74bc12b4b6372] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp, totalSize=92.6 K 2023-09-29T06:15:59,786 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/75aec740ea35434b9928b32008e6b1ad, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/40ec9a7f5e074a9a87e606f54cf372d7, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/7c66252782774860bd09b51b88ac19e9] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp, totalSize=37.1 K 2023-09-29T06:15:59,786 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:59,787 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. files: [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/24a943df07aa48c6831ede40991666f3, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/dec21e52bb8742bf9d59bb26bda389ad, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/ac8307b719234b129ea74bc12b4b6372] 2023-09-29T06:15:59,787 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 75aec740ea35434b9928b32008e6b1ad, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=489, earliestPutTs=1695968155530 2023-09-29T06:15:59,787 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting 24a943df07aa48c6831ede40991666f3, keycount=150, bloomtype=ROW, size=31.6 K, encoding=NONE, compression=NONE, seqNum=489, earliestPutTs=1695968155530 2023-09-29T06:15:59,787 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 40ec9a7f5e074a9a87e606f54cf372d7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=515, earliestPutTs=1695968155889 2023-09-29T06:15:59,787 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting dec21e52bb8742bf9d59bb26bda389ad, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=515, earliestPutTs=1695968155889 2023-09-29T06:15:59,787 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 7c66252782774860bd09b51b88ac19e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=529, earliestPutTs=1695968157017 2023-09-29T06:15:59,787 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] compactions.Compactor(223): Compacting ac8307b719234b129ea74bc12b4b6372, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=529, earliestPutTs=1695968157017 2023-09-29T06:15:59,792 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:59,792 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f1c71e3f887ce47e67c17dad51fd610#B#compaction#659 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:59,793 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/df80abd6c46847ad81856a7098ed789c is 50, key is test_row_0/B:col10/1695968157017/Put/seqid=0 2023-09-29T06:15:59,793 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202309297271ccd386c647ab9be4eda935c26876_0f1c71e3f887ce47e67c17dad51fd610 store=[table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:59,794 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202309297271ccd386c647ab9be4eda935c26876_0f1c71e3f887ce47e67c17dad51fd610, store=[table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:59,795 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202309297271ccd386c647ab9be4eda935c26876_0f1c71e3f887ce47e67c17dad51fd610 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:15:59,800 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/df80abd6c46847ad81856a7098ed789c as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/df80abd6c46847ad81856a7098ed789c 2023-09-29T06:15:59,804 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 0f1c71e3f887ce47e67c17dad51fd610/B of 0f1c71e3f887ce47e67c17dad51fd610 into df80abd6c46847ad81856a7098ed789c(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:15:59,804 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:15:59,804 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610., storeName=0f1c71e3f887ce47e67c17dad51fd610/B, priority=13, startTime=1695968159785; duration=0sec 2023-09-29T06:15:59,804 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2023-09-29T06:15:59,804 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f1c71e3f887ce47e67c17dad51fd610:B 2023-09-29T06:15:59,804 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-29T06:15:59,804 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38010 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-29T06:15:59,805 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1534): 0f1c71e3f887ce47e67c17dad51fd610/C is initiating minor compaction (all files) 2023-09-29T06:15:59,805 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 0f1c71e3f887ce47e67c17dad51fd610/C in TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:15:59,805 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/1f364328b01b4832aa7f7128083bdcda, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/ff12dde1ecee4d7284c1705a5eb72fd6, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/c2dc7aea006641f4971b3871f1930104] into tmpdir=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp, totalSize=37.1 K 2023-09-29T06:15:59,805 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting 1f364328b01b4832aa7f7128083bdcda, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=489, earliestPutTs=1695968155530 2023-09-29T06:15:59,805 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting ff12dde1ecee4d7284c1705a5eb72fd6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=515, earliestPutTs=1695968155889 2023-09-29T06:15:59,805 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] compactions.Compactor(223): Compacting c2dc7aea006641f4971b3871f1930104, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=529, earliestPutTs=1695968157017 2023-09-29T06:15:59,810 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f1c71e3f887ce47e67c17dad51fd610#C#compaction#661 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:15:59,810 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/cdb8762017a54830a573873085aca3b9 is 50, key is test_row_0/C:col10/1695968157017/Put/seqid=0 2023-09-29T06:16:00,199 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f1c71e3f887ce47e67c17dad51fd610#A#compaction#660 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-29T06:16:00,200 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/e2181e7bfe544ddda3070103642e679a is 175, key is test_row_0/A:col10/1695968157017/Put/seqid=0 2023-09-29T06:16:00,216 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/cdb8762017a54830a573873085aca3b9 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/cdb8762017a54830a573873085aca3b9 2023-09-29T06:16:00,220 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 0f1c71e3f887ce47e67c17dad51fd610/C of 0f1c71e3f887ce47e67c17dad51fd610 into cdb8762017a54830a573873085aca3b9(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:16:00,220 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:16:00,220 INFO [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610., storeName=0f1c71e3f887ce47e67c17dad51fd610/C, priority=13, startTime=1695968159785; duration=0sec 2023-09-29T06:16:00,220 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:16:00,220 DEBUG [RS:0;jenkins-hbase3:39783-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f1c71e3f887ce47e67c17dad51fd610:C 2023-09-29T06:16:00,607 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/e2181e7bfe544ddda3070103642e679a as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/e2181e7bfe544ddda3070103642e679a 2023-09-29T06:16:00,610 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 0f1c71e3f887ce47e67c17dad51fd610/A of 0f1c71e3f887ce47e67c17dad51fd610 into e2181e7bfe544ddda3070103642e679a(size=31.7 K), total size for store is 31.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-29T06:16:00,610 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:16:00,610 INFO [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610., storeName=0f1c71e3f887ce47e67c17dad51fd610/A, priority=13, startTime=1695968159785; duration=0sec 2023-09-29T06:16:00,610 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-29T06:16:00,610 DEBUG [RS:0;jenkins-hbase3:39783-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f1c71e3f887ce47e67c17dad51fd610:A 2023-09-29T06:16:05,648 DEBUG [Thread-3330 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0d1a704d to 127.0.0.1:49344 2023-09-29T06:16:05,648 DEBUG [Thread-3330 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:16:05,671 DEBUG [Thread-3326 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x5e13bd56 to 127.0.0.1:49344 2023-09-29T06:16:05,671 DEBUG [Thread-3326 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:16:05,679 DEBUG [Thread-3328 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x48436ecd to 127.0.0.1:49344 2023-09-29T06:16:05,679 DEBUG [Thread-3328 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:16:05,704 DEBUG [Thread-3324 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x2aaa0778 to 127.0.0.1:49344 2023-09-29T06:16:05,704 DEBUG [Thread-3324 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:16:05,704 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2023-09-29T06:16:05,704 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(394): wrote 208 2023-09-29T06:16:05,704 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2023-09-29T06:16:05,705 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2023-09-29T06:16:05,705 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2023-09-29T06:16:05,705 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2023-09-29T06:16:05,705 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(396): Readers: 2023-09-29T06:16:05,705 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(398): read 7555 2023-09-29T06:16:05,705 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(398): read 7365 2023-09-29T06:16:05,705 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(398): read 7463 2023-09-29T06:16:05,705 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(398): read 7539 2023-09-29T06:16:05,705 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(398): read 7329 2023-09-29T06:16:05,705 INFO [Listener at localhost/42843 {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2023-09-29T06:16:05,705 INFO [Listener at localhost/42843 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-09-29T06:16:05,705 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x522ff045 to 127.0.0.1:49344 2023-09-29T06:16:05,705 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:16:05,705 INFO [Listener at localhost/42843 {}] client.HBaseAdmin$18(967): Started disable of TestAcidGuarantees 2023-09-29T06:16:05,706 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster$12(2664): Client=jenkins//172.31.12.81 disable TestAcidGuarantees 2023-09-29T06:16:05,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=194, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=TestAcidGuarantees 2023-09-29T06:16:05,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=194 2023-09-29T06:16:05,708 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968165708"}]},"ts":"1695968165708"} 2023-09-29T06:16:05,709 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2023-09-29T06:16:05,711 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(293): Set TestAcidGuarantees to state=DISABLING 2023-09-29T06:16:05,711 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=195, ppid=194, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=0f1c71e3f887ce47e67c17dad51fd610, UNASSIGN}] 2023-09-29T06:16:05,712 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=195, ppid=194, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=0f1c71e3f887ce47e67c17dad51fd610, UNASSIGN 2023-09-29T06:16:05,713 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=195 updating hbase:meta row=0f1c71e3f887ce47e67c17dad51fd610, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:16:05,714 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=196, ppid=195, state=RUNNABLE; CloseRegionProcedure 0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138}] 2023-09-29T06:16:05,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=194 2023-09-29T06:16:05,865 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=196}] handler.UnassignRegionHandler(119): Close 0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:16:05,865 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=196}] regionserver.HRegion(1668): Closing 0f1c71e3f887ce47e67c17dad51fd610, disabling compactions & flushes 2023-09-29T06:16:05,865 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=196}] regionserver.HRegion(1690): Closing region TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:16:05,865 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=196}] regionserver.HRegion(1711): Waiting without time limit for close lock on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:16:05,865 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=196}] regionserver.HRegion(1778): Acquired close lock on TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. after waiting 0 ms 2023-09-29T06:16:05,865 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=196}] regionserver.HRegion(1788): Updates disabled for region TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:16:05,865 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=196}] regionserver.HRegion(2819): Flushing 0f1c71e3f887ce47e67c17dad51fd610 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2023-09-29T06:16:05,866 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=196}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=A 2023-09-29T06:16:05,866 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=196}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:16:05,866 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=196}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=B 2023-09-29T06:16:05,866 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=196}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:16:05,866 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=196}] regionserver.CompactingMemStore(204): FLUSHING TO DISK 0f1c71e3f887ce47e67c17dad51fd610, store=C 2023-09-29T06:16:05,866 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=196}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2023-09-29T06:16:05,869 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=196}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929966c67c204f64fce8a12863bcd47332b_0f1c71e3f887ce47e67c17dad51fd610 is 50, key is test_row_0/A:col10/1695968165703/Put/seqid=0 2023-09-29T06:16:06,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=194 2023-09-29T06:16:06,275 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=196}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20230929966c67c204f64fce8a12863bcd47332b_0f1c71e3f887ce47e67c17dad51fd610 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929966c67c204f64fce8a12863bcd47332b_0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:16:06,276 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=196}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/92e4bbcbcf40425a828b3b91a609b45f, store: [table=TestAcidGuarantees family=A region=0f1c71e3f887ce47e67c17dad51fd610] 2023-09-29T06:16:06,277 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=196}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/92e4bbcbcf40425a828b3b91a609b45f is 175, key is test_row_0/A:col10/1695968165703/Put/seqid=0 2023-09-29T06:16:06,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=194 2023-09-29T06:16:06,680 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=196}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=539, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/92e4bbcbcf40425a828b3b91a609b45f 2023-09-29T06:16:06,685 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=196}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/d6983ad056e64dd1a6ad28728dce7986 is 50, key is test_row_0/B:col10/1695968165703/Put/seqid=0 2023-09-29T06:16:06,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=194 2023-09-29T06:16:07,088 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=196}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=539 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/d6983ad056e64dd1a6ad28728dce7986 2023-09-29T06:16:07,092 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=196}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/2d0dffafa5274b218ed01c260f102ea8 is 50, key is test_row_0/C:col10/1695968165703/Put/seqid=0 2023-09-29T06:16:07,496 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=196}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=539 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/2d0dffafa5274b218ed01c260f102ea8 2023-09-29T06:16:07,500 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=196}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/A/92e4bbcbcf40425a828b3b91a609b45f as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/92e4bbcbcf40425a828b3b91a609b45f 2023-09-29T06:16:07,503 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=196}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/92e4bbcbcf40425a828b3b91a609b45f, entries=100, sequenceid=539, filesize=22.0 K 2023-09-29T06:16:07,503 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=196}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/B/d6983ad056e64dd1a6ad28728dce7986 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/d6983ad056e64dd1a6ad28728dce7986 2023-09-29T06:16:07,506 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=196}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/d6983ad056e64dd1a6ad28728dce7986, entries=100, sequenceid=539, filesize=9.6 K 2023-09-29T06:16:07,507 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=196}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/.tmp/C/2d0dffafa5274b218ed01c260f102ea8 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/2d0dffafa5274b218ed01c260f102ea8 2023-09-29T06:16:07,509 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=196}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/2d0dffafa5274b218ed01c260f102ea8, entries=100, sequenceid=539, filesize=9.6 K 2023-09-29T06:16:07,510 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=196}] regionserver.HRegion(3022): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 0f1c71e3f887ce47e67c17dad51fd610 in 1645ms, sequenceid=539, compaction requested=false 2023-09-29T06:16:07,511 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/a622c762a94d40f780e68341e1832272, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/fd247fbbced14cb4943621fa4545acaa, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/7694fb17648442cf89a7ee43dd4d3912, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/f8cb2c5594b84ff8ba125ba82c586b4d, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/c48f39dd900a4714ab3fcebe81c4a5ef, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/b26bc88f54fd484da3e4680f6bb4fa54, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/945d293141324abeb220fa4c31756407, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/56f139aed7f644f3b045cd5efdba189f, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/7fa34e860e2442c1ac8a8cd2ff5aec29, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/b6b81bba26da41a9b3170540423796e7, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/81b5ed1eea654b858e01ec80e7fe60c7, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/280b325d11fc4664a1a9b763be26b194, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/f89b4abed9dd48858f77fa28346352d4, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/51cc6b734dc34c06adcd2e56e0b5ba2b, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/21374984b6a441a2afba300510ad1b11, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/206ce229acde432384e3898c2ddababd, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/11dd37f357cf4f948af57c27bb327125, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/59ece13175e9451985555188b15c9c35, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/a40f66022f054d33a3317c37714ae2bc, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/79850abb8b50400dbd10538be38f4a6c, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/c1eff5b143814c51aefdcebaa1c865c8, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/37a13dccf6334906a4b33a3a47fcf33e, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/b9144deae82e4bf9a5c01407faeb0c7c, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/a11a931452fb4e4a993c01a9844c8eec, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/066be9bd9396442e8330c84cb48b6077, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/396d3914646e47ee8e75015e5865b256, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/51ab26f526764f0f806ba8d06fff2570, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/48754350c6814b558e94e6ebd55ba15a, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/a9830e5929114a4597ed597c609560af, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/bee1c85ccbc842f39896798d3b86e24e, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/83b97c4249d149ff874c1015454529a3, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/4ec3ec9b066b4c1c96528957d72cb685, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/54dafbd31cbb498f8ebcf452691117c4, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/24a943df07aa48c6831ede40991666f3, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/50648b3f04924881b0201f08431112e1, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/dec21e52bb8742bf9d59bb26bda389ad, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/ac8307b719234b129ea74bc12b4b6372] to archive 2023-09-29T06:16:07,512 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2023-09-29T06:16:07,513 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/a622c762a94d40f780e68341e1832272 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/a622c762a94d40f780e68341e1832272 2023-09-29T06:16:07,514 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/fd247fbbced14cb4943621fa4545acaa to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/fd247fbbced14cb4943621fa4545acaa 2023-09-29T06:16:07,515 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/7694fb17648442cf89a7ee43dd4d3912 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/7694fb17648442cf89a7ee43dd4d3912 2023-09-29T06:16:07,516 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/f8cb2c5594b84ff8ba125ba82c586b4d to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/f8cb2c5594b84ff8ba125ba82c586b4d 2023-09-29T06:16:07,517 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/c48f39dd900a4714ab3fcebe81c4a5ef to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/c48f39dd900a4714ab3fcebe81c4a5ef 2023-09-29T06:16:07,517 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/b26bc88f54fd484da3e4680f6bb4fa54 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/b26bc88f54fd484da3e4680f6bb4fa54 2023-09-29T06:16:07,518 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/945d293141324abeb220fa4c31756407 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/945d293141324abeb220fa4c31756407 2023-09-29T06:16:07,519 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/56f139aed7f644f3b045cd5efdba189f to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/56f139aed7f644f3b045cd5efdba189f 2023-09-29T06:16:07,520 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/7fa34e860e2442c1ac8a8cd2ff5aec29 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/7fa34e860e2442c1ac8a8cd2ff5aec29 2023-09-29T06:16:07,521 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/b6b81bba26da41a9b3170540423796e7 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/b6b81bba26da41a9b3170540423796e7 2023-09-29T06:16:07,522 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/81b5ed1eea654b858e01ec80e7fe60c7 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/81b5ed1eea654b858e01ec80e7fe60c7 2023-09-29T06:16:07,523 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/280b325d11fc4664a1a9b763be26b194 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/280b325d11fc4664a1a9b763be26b194 2023-09-29T06:16:07,524 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/f89b4abed9dd48858f77fa28346352d4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/f89b4abed9dd48858f77fa28346352d4 2023-09-29T06:16:07,524 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/51cc6b734dc34c06adcd2e56e0b5ba2b to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/51cc6b734dc34c06adcd2e56e0b5ba2b 2023-09-29T06:16:07,525 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/21374984b6a441a2afba300510ad1b11 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/21374984b6a441a2afba300510ad1b11 2023-09-29T06:16:07,526 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/206ce229acde432384e3898c2ddababd to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/206ce229acde432384e3898c2ddababd 2023-09-29T06:16:07,527 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/11dd37f357cf4f948af57c27bb327125 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/11dd37f357cf4f948af57c27bb327125 2023-09-29T06:16:07,528 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/59ece13175e9451985555188b15c9c35 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/59ece13175e9451985555188b15c9c35 2023-09-29T06:16:07,528 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/a40f66022f054d33a3317c37714ae2bc to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/a40f66022f054d33a3317c37714ae2bc 2023-09-29T06:16:07,529 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/79850abb8b50400dbd10538be38f4a6c to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/79850abb8b50400dbd10538be38f4a6c 2023-09-29T06:16:07,530 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/c1eff5b143814c51aefdcebaa1c865c8 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/c1eff5b143814c51aefdcebaa1c865c8 2023-09-29T06:16:07,531 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/37a13dccf6334906a4b33a3a47fcf33e to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/37a13dccf6334906a4b33a3a47fcf33e 2023-09-29T06:16:07,532 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/b9144deae82e4bf9a5c01407faeb0c7c to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/b9144deae82e4bf9a5c01407faeb0c7c 2023-09-29T06:16:07,533 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/a11a931452fb4e4a993c01a9844c8eec to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/a11a931452fb4e4a993c01a9844c8eec 2023-09-29T06:16:07,533 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/066be9bd9396442e8330c84cb48b6077 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/066be9bd9396442e8330c84cb48b6077 2023-09-29T06:16:07,534 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/396d3914646e47ee8e75015e5865b256 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/396d3914646e47ee8e75015e5865b256 2023-09-29T06:16:07,535 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/51ab26f526764f0f806ba8d06fff2570 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/51ab26f526764f0f806ba8d06fff2570 2023-09-29T06:16:07,536 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/48754350c6814b558e94e6ebd55ba15a to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/48754350c6814b558e94e6ebd55ba15a 2023-09-29T06:16:07,537 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/a9830e5929114a4597ed597c609560af to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/a9830e5929114a4597ed597c609560af 2023-09-29T06:16:07,537 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/bee1c85ccbc842f39896798d3b86e24e to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/bee1c85ccbc842f39896798d3b86e24e 2023-09-29T06:16:07,538 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/83b97c4249d149ff874c1015454529a3 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/83b97c4249d149ff874c1015454529a3 2023-09-29T06:16:07,539 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/4ec3ec9b066b4c1c96528957d72cb685 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/4ec3ec9b066b4c1c96528957d72cb685 2023-09-29T06:16:07,540 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/54dafbd31cbb498f8ebcf452691117c4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/54dafbd31cbb498f8ebcf452691117c4 2023-09-29T06:16:07,541 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/24a943df07aa48c6831ede40991666f3 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/24a943df07aa48c6831ede40991666f3 2023-09-29T06:16:07,541 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/50648b3f04924881b0201f08431112e1 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/50648b3f04924881b0201f08431112e1 2023-09-29T06:16:07,542 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/dec21e52bb8742bf9d59bb26bda389ad to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/dec21e52bb8742bf9d59bb26bda389ad 2023-09-29T06:16:07,543 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/ac8307b719234b129ea74bc12b4b6372 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/ac8307b719234b129ea74bc12b4b6372 2023-09-29T06:16:07,546 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/fbbbe3e374d94521ada24a6de6c2663d, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/92e6bd729ea047f5a63c98d391b05472, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/e426edc6f3a043a4897c68127d7ca87f, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/79341a55e74f4e0689d60bc302ade5fd, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/55e5b7fe941f49a494352663a96e0489, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/483326bf15c14cc1a558c9a005468583, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/9a10fab4dc744666bc8bcd1574b03e7f, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/cd479ef3042144dc995a7918bc021098, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/3e4e7ecdb8014740b85dd12f3313f99e, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/1164583a760d4bf9aed78945ddbb82b6, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/60f3e663bc4f47838fa5b5c9b9c86fde, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/b7176e59f8f94c3db61fc08e1fd7d7a4, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/8524bdbefe964c6db86a66eb265b7621, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/ec218e2d29754a7cbda1ee33800e6cfe, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/12d81dff835048848f8d0c554a90f88b, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/600c823c598543a9b74283f0d83671b3, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/d077d7557734444fa9375e8e5ced921f, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/da3fd91a881a4d7d99c3a326a87cdc74, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/a8f8afa8735e47fc9ba84f0364ab0533, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/23e9e8b7bff14a9b90f0571a168bea31, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/6d6326409315494fa12fdca35314d343, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/69361123921b4d6ea5c573afe22b211c, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/913626044c0b4676992d400b1e0e1c41, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/24a500531dc1459682e20f9ca9489076, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/d2bbf0f79bfe4420b2f7873f35c25ce8, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/39962d56e21d48a283d62779b8a55d4c, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/2312631cad704f38bee91b890f7b81a0, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/f965076f91bf469fa3e4f3d3fd86c2ed, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/ce9d9b2624fb41819b5d7e264266d358, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/04455329bb7f4d16a27b3188ce6a2a7c, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/b7bbd4104f8f49fab29084e66d49c543, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/25650ae36f3341ddade2842bc8e7f899, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/939dca46d14d40e1b0b440a842645865, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/75aec740ea35434b9928b32008e6b1ad, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/301f1393a4fe4405aa2344e4386099d3, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/40ec9a7f5e074a9a87e606f54cf372d7, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/7c66252782774860bd09b51b88ac19e9] to archive 2023-09-29T06:16:07,547 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2023-09-29T06:16:07,548 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/fbbbe3e374d94521ada24a6de6c2663d to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/fbbbe3e374d94521ada24a6de6c2663d 2023-09-29T06:16:07,549 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/92e6bd729ea047f5a63c98d391b05472 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/92e6bd729ea047f5a63c98d391b05472 2023-09-29T06:16:07,549 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/e426edc6f3a043a4897c68127d7ca87f to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/e426edc6f3a043a4897c68127d7ca87f 2023-09-29T06:16:07,550 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/79341a55e74f4e0689d60bc302ade5fd to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/79341a55e74f4e0689d60bc302ade5fd 2023-09-29T06:16:07,551 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/55e5b7fe941f49a494352663a96e0489 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/55e5b7fe941f49a494352663a96e0489 2023-09-29T06:16:07,552 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/483326bf15c14cc1a558c9a005468583 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/483326bf15c14cc1a558c9a005468583 2023-09-29T06:16:07,553 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/9a10fab4dc744666bc8bcd1574b03e7f to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/9a10fab4dc744666bc8bcd1574b03e7f 2023-09-29T06:16:07,554 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/cd479ef3042144dc995a7918bc021098 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/cd479ef3042144dc995a7918bc021098 2023-09-29T06:16:07,555 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/3e4e7ecdb8014740b85dd12f3313f99e to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/3e4e7ecdb8014740b85dd12f3313f99e 2023-09-29T06:16:07,555 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/1164583a760d4bf9aed78945ddbb82b6 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/1164583a760d4bf9aed78945ddbb82b6 2023-09-29T06:16:07,556 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/60f3e663bc4f47838fa5b5c9b9c86fde to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/60f3e663bc4f47838fa5b5c9b9c86fde 2023-09-29T06:16:07,557 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/b7176e59f8f94c3db61fc08e1fd7d7a4 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/b7176e59f8f94c3db61fc08e1fd7d7a4 2023-09-29T06:16:07,558 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/8524bdbefe964c6db86a66eb265b7621 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/8524bdbefe964c6db86a66eb265b7621 2023-09-29T06:16:07,559 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/ec218e2d29754a7cbda1ee33800e6cfe to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/ec218e2d29754a7cbda1ee33800e6cfe 2023-09-29T06:16:07,560 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/12d81dff835048848f8d0c554a90f88b to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/12d81dff835048848f8d0c554a90f88b 2023-09-29T06:16:07,560 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/600c823c598543a9b74283f0d83671b3 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/600c823c598543a9b74283f0d83671b3 2023-09-29T06:16:07,561 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/d077d7557734444fa9375e8e5ced921f to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/d077d7557734444fa9375e8e5ced921f 2023-09-29T06:16:07,562 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/da3fd91a881a4d7d99c3a326a87cdc74 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/da3fd91a881a4d7d99c3a326a87cdc74 2023-09-29T06:16:07,563 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/a8f8afa8735e47fc9ba84f0364ab0533 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/a8f8afa8735e47fc9ba84f0364ab0533 2023-09-29T06:16:07,564 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/23e9e8b7bff14a9b90f0571a168bea31 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/23e9e8b7bff14a9b90f0571a168bea31 2023-09-29T06:16:07,564 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/6d6326409315494fa12fdca35314d343 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/6d6326409315494fa12fdca35314d343 2023-09-29T06:16:07,565 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/69361123921b4d6ea5c573afe22b211c to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/69361123921b4d6ea5c573afe22b211c 2023-09-29T06:16:07,566 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/913626044c0b4676992d400b1e0e1c41 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/913626044c0b4676992d400b1e0e1c41 2023-09-29T06:16:07,567 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/24a500531dc1459682e20f9ca9489076 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/24a500531dc1459682e20f9ca9489076 2023-09-29T06:16:07,567 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/d2bbf0f79bfe4420b2f7873f35c25ce8 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/d2bbf0f79bfe4420b2f7873f35c25ce8 2023-09-29T06:16:07,568 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/39962d56e21d48a283d62779b8a55d4c to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/39962d56e21d48a283d62779b8a55d4c 2023-09-29T06:16:07,569 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/2312631cad704f38bee91b890f7b81a0 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/2312631cad704f38bee91b890f7b81a0 2023-09-29T06:16:07,570 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/f965076f91bf469fa3e4f3d3fd86c2ed to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/f965076f91bf469fa3e4f3d3fd86c2ed 2023-09-29T06:16:07,571 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/ce9d9b2624fb41819b5d7e264266d358 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/ce9d9b2624fb41819b5d7e264266d358 2023-09-29T06:16:07,571 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/04455329bb7f4d16a27b3188ce6a2a7c to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/04455329bb7f4d16a27b3188ce6a2a7c 2023-09-29T06:16:07,572 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/b7bbd4104f8f49fab29084e66d49c543 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/b7bbd4104f8f49fab29084e66d49c543 2023-09-29T06:16:07,573 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/25650ae36f3341ddade2842bc8e7f899 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/25650ae36f3341ddade2842bc8e7f899 2023-09-29T06:16:07,573 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/939dca46d14d40e1b0b440a842645865 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/939dca46d14d40e1b0b440a842645865 2023-09-29T06:16:07,574 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/75aec740ea35434b9928b32008e6b1ad to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/75aec740ea35434b9928b32008e6b1ad 2023-09-29T06:16:07,575 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/301f1393a4fe4405aa2344e4386099d3 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/301f1393a4fe4405aa2344e4386099d3 2023-09-29T06:16:07,576 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/40ec9a7f5e074a9a87e606f54cf372d7 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/40ec9a7f5e074a9a87e606f54cf372d7 2023-09-29T06:16:07,576 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/7c66252782774860bd09b51b88ac19e9 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/7c66252782774860bd09b51b88ac19e9 2023-09-29T06:16:07,579 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/8461659db7624ed1a514a4352dc5db97, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/a9ccb23f36ca4a94a8af8bfc70110569, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/664ed9a98df24e5e8313dd1f8525ce2a, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/9e04a716f69c47d1bb99aa134d2e67ab, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/76722bf8fce641b2b1bbba24159556cf, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/b4a901048ab84f9bb07e6cbfbe5ca4f9, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/f168994376664737966c0f92d02dde3f, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/94572942f9ed4009a6a5e489995919fd, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/4652fc5074d14c04bd7b977077b2a07d, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/7ed4f6649311440d92235d85a8919148, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/f55aac84da7e4d76aff17c314da3130d, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/a19545db00904b188800ad44d04de9bf, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/9135b3c0464c4c61be979933f9aa89f5, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/a6d3f9cd54ad4ac0bff1ac74556ad864, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/30fc870dd8f94f058bfad7573205e596, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/8896e151a20344ce9a20affce47ad963, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/4b8309695b3d4470bba8117dbdea6970, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/d1974cf42b46469188e2455b0a1390a1, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/d5c61c506034478ea7e5bb202bd6791b, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/2871c6d06c394836831f2f4bcbf500b3, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/2726e80b875c4209a4d12a97a1dbbc90, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/093a2aea215446ffb95834425edcd8a9, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/b6df0ef94a144f268e79f1b1586a09cd, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/d6cfc457cb0e45ccb946f96971be1f78, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/7ce3c32c82704ae785c089450cff05b7, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/4c0702689bb74a41bf4ad2190443bdf5, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/caee7110a4c74bf8bd5138fe45167f37, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/f7ac435bb0fa49138d2d1e75e80d2f4e, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/ecb46689dbdf405bb728219b1fe7b6b0, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/f912d9216a61451c86e999c68a6225d9, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/0140649f8e214161ba2dca673c9f9564, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/1676483468344f1c94b442f176a9cf33, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/f3fb36313ef04ed6a9d268673cd492b5, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/1f364328b01b4832aa7f7128083bdcda, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/8d43366393c84aa4a1b5327599fb2cba, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/ff12dde1ecee4d7284c1705a5eb72fd6, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/c2dc7aea006641f4971b3871f1930104] to archive 2023-09-29T06:16:07,580 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2023-09-29T06:16:07,581 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/8461659db7624ed1a514a4352dc5db97 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/8461659db7624ed1a514a4352dc5db97 2023-09-29T06:16:07,582 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/a9ccb23f36ca4a94a8af8bfc70110569 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/a9ccb23f36ca4a94a8af8bfc70110569 2023-09-29T06:16:07,583 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/664ed9a98df24e5e8313dd1f8525ce2a to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/664ed9a98df24e5e8313dd1f8525ce2a 2023-09-29T06:16:07,583 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/9e04a716f69c47d1bb99aa134d2e67ab to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/9e04a716f69c47d1bb99aa134d2e67ab 2023-09-29T06:16:07,584 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/76722bf8fce641b2b1bbba24159556cf to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/76722bf8fce641b2b1bbba24159556cf 2023-09-29T06:16:07,585 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/b4a901048ab84f9bb07e6cbfbe5ca4f9 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/b4a901048ab84f9bb07e6cbfbe5ca4f9 2023-09-29T06:16:07,586 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/f168994376664737966c0f92d02dde3f to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/f168994376664737966c0f92d02dde3f 2023-09-29T06:16:07,586 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/94572942f9ed4009a6a5e489995919fd to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/94572942f9ed4009a6a5e489995919fd 2023-09-29T06:16:07,587 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/4652fc5074d14c04bd7b977077b2a07d to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/4652fc5074d14c04bd7b977077b2a07d 2023-09-29T06:16:07,588 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/7ed4f6649311440d92235d85a8919148 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/7ed4f6649311440d92235d85a8919148 2023-09-29T06:16:07,588 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/f55aac84da7e4d76aff17c314da3130d to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/f55aac84da7e4d76aff17c314da3130d 2023-09-29T06:16:07,589 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/a19545db00904b188800ad44d04de9bf to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/a19545db00904b188800ad44d04de9bf 2023-09-29T06:16:07,590 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/9135b3c0464c4c61be979933f9aa89f5 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/9135b3c0464c4c61be979933f9aa89f5 2023-09-29T06:16:07,591 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/a6d3f9cd54ad4ac0bff1ac74556ad864 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/a6d3f9cd54ad4ac0bff1ac74556ad864 2023-09-29T06:16:07,591 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/30fc870dd8f94f058bfad7573205e596 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/30fc870dd8f94f058bfad7573205e596 2023-09-29T06:16:07,592 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/8896e151a20344ce9a20affce47ad963 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/8896e151a20344ce9a20affce47ad963 2023-09-29T06:16:07,593 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/4b8309695b3d4470bba8117dbdea6970 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/4b8309695b3d4470bba8117dbdea6970 2023-09-29T06:16:07,594 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/d1974cf42b46469188e2455b0a1390a1 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/d1974cf42b46469188e2455b0a1390a1 2023-09-29T06:16:07,595 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/d5c61c506034478ea7e5bb202bd6791b to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/d5c61c506034478ea7e5bb202bd6791b 2023-09-29T06:16:07,595 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/2871c6d06c394836831f2f4bcbf500b3 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/2871c6d06c394836831f2f4bcbf500b3 2023-09-29T06:16:07,596 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/2726e80b875c4209a4d12a97a1dbbc90 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/2726e80b875c4209a4d12a97a1dbbc90 2023-09-29T06:16:07,597 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/093a2aea215446ffb95834425edcd8a9 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/093a2aea215446ffb95834425edcd8a9 2023-09-29T06:16:07,598 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/b6df0ef94a144f268e79f1b1586a09cd to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/b6df0ef94a144f268e79f1b1586a09cd 2023-09-29T06:16:07,598 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/d6cfc457cb0e45ccb946f96971be1f78 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/d6cfc457cb0e45ccb946f96971be1f78 2023-09-29T06:16:07,599 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/7ce3c32c82704ae785c089450cff05b7 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/7ce3c32c82704ae785c089450cff05b7 2023-09-29T06:16:07,600 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/4c0702689bb74a41bf4ad2190443bdf5 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/4c0702689bb74a41bf4ad2190443bdf5 2023-09-29T06:16:07,600 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/caee7110a4c74bf8bd5138fe45167f37 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/caee7110a4c74bf8bd5138fe45167f37 2023-09-29T06:16:07,601 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/f7ac435bb0fa49138d2d1e75e80d2f4e to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/f7ac435bb0fa49138d2d1e75e80d2f4e 2023-09-29T06:16:07,602 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/ecb46689dbdf405bb728219b1fe7b6b0 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/ecb46689dbdf405bb728219b1fe7b6b0 2023-09-29T06:16:07,603 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/f912d9216a61451c86e999c68a6225d9 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/f912d9216a61451c86e999c68a6225d9 2023-09-29T06:16:07,604 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/0140649f8e214161ba2dca673c9f9564 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/0140649f8e214161ba2dca673c9f9564 2023-09-29T06:16:07,604 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/1676483468344f1c94b442f176a9cf33 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/1676483468344f1c94b442f176a9cf33 2023-09-29T06:16:07,605 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/f3fb36313ef04ed6a9d268673cd492b5 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/f3fb36313ef04ed6a9d268673cd492b5 2023-09-29T06:16:07,606 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/1f364328b01b4832aa7f7128083bdcda to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/1f364328b01b4832aa7f7128083bdcda 2023-09-29T06:16:07,607 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/8d43366393c84aa4a1b5327599fb2cba to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/8d43366393c84aa4a1b5327599fb2cba 2023-09-29T06:16:07,608 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/ff12dde1ecee4d7284c1705a5eb72fd6 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/ff12dde1ecee4d7284c1705a5eb72fd6 2023-09-29T06:16:07,609 DEBUG [StoreCloser-TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/c2dc7aea006641f4971b3871f1930104 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/c2dc7aea006641f4971b3871f1930104 2023-09-29T06:16:07,613 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=196}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/recovered.edits/542.seqid, newMaxSeqId=542, maxSeqId=4 2023-09-29T06:16:07,614 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=196}] regionserver.HRegion(1909): Closed TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610. 2023-09-29T06:16:07,614 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=196}] regionserver.HRegion(1622): Region close journal for 0f1c71e3f887ce47e67c17dad51fd610: 2023-09-29T06:16:07,615 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=196}] handler.UnassignRegionHandler(163): Closed 0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:16:07,615 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=195 updating hbase:meta row=0f1c71e3f887ce47e67c17dad51fd610, regionState=CLOSED 2023-09-29T06:16:07,617 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=196, resume processing ppid=195 2023-09-29T06:16:07,617 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=196, ppid=195, state=SUCCESS; CloseRegionProcedure 0f1c71e3f887ce47e67c17dad51fd610, server=jenkins-hbase3.apache.org,39783,1695967978138 in 1.9020 sec 2023-09-29T06:16:07,618 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=195, resume processing ppid=194 2023-09-29T06:16:07,618 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=195, ppid=194, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=0f1c71e3f887ce47e67c17dad51fd610, UNASSIGN in 1.9060 sec 2023-09-29T06:16:07,619 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1695968167619"}]},"ts":"1695968167619"} 2023-09-29T06:16:07,620 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2023-09-29T06:16:07,622 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(305): Set TestAcidGuarantees to state=DISABLED 2023-09-29T06:16:07,623 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=194, state=SUCCESS; DisableTableProcedure table=TestAcidGuarantees in 1.9160 sec 2023-09-29T06:16:07,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=194 2023-09-29T06:16:07,813 INFO [Listener at localhost/42843 {}] client.HBaseAdmin$TableFuture(3677): Operation: DISABLE, Table Name: default:TestAcidGuarantees, procId: 194 completed 2023-09-29T06:16:07,814 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.HMaster$5(2448): Client=jenkins//172.31.12.81 delete TestAcidGuarantees 2023-09-29T06:16:07,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] procedure2.ProcedureExecutor(1032): Stored pid=197, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=TestAcidGuarantees 2023-09-29T06:16:07,816 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=197, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2023-09-29T06:16:07,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=197 2023-09-29T06:16:07,816 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=197, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2023-09-29T06:16:07,819 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:16:07,821 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A, FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B, FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C, FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/recovered.edits] 2023-09-29T06:16:07,823 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/92e4bbcbcf40425a828b3b91a609b45f to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/92e4bbcbcf40425a828b3b91a609b45f 2023-09-29T06:16:07,824 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/e2181e7bfe544ddda3070103642e679a to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/A/e2181e7bfe544ddda3070103642e679a 2023-09-29T06:16:07,826 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/d6983ad056e64dd1a6ad28728dce7986 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/d6983ad056e64dd1a6ad28728dce7986 2023-09-29T06:16:07,827 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/df80abd6c46847ad81856a7098ed789c to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/B/df80abd6c46847ad81856a7098ed789c 2023-09-29T06:16:07,829 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/2d0dffafa5274b218ed01c260f102ea8 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/2d0dffafa5274b218ed01c260f102ea8 2023-09-29T06:16:07,830 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/cdb8762017a54830a573873085aca3b9 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/C/cdb8762017a54830a573873085aca3b9 2023-09-29T06:16:07,832 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/recovered.edits/542.seqid to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610/recovered.edits/542.seqid 2023-09-29T06:16:07,833 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/default/TestAcidGuarantees/0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:16:07,833 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(293): Archived TestAcidGuarantees regions 2023-09-29T06:16:07,833 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2023-09-29T06:16:07,833 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2023-09-29T06:16:07,836 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023092900e70be28e5d4aec9a04c7b434b03d15_0f1c71e3f887ce47e67c17dad51fd610 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023092900e70be28e5d4aec9a04c7b434b03d15_0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:16:07,837 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309290688eadd0f1a450781edba7ad4533531_0f1c71e3f887ce47e67c17dad51fd610 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309290688eadd0f1a450781edba7ad4533531_0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:16:07,837 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309290f8ab063fc1541c6a1629ceaf2a53b8a_0f1c71e3f887ce47e67c17dad51fd610 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309290f8ab063fc1541c6a1629ceaf2a53b8a_0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:16:07,838 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023092912acc22587314b089a5e0e4e48b0eaa6_0f1c71e3f887ce47e67c17dad51fd610 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023092912acc22587314b089a5e0e4e48b0eaa6_0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:16:07,839 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309291489e7e936e94b73a6609d582671a15e_0f1c71e3f887ce47e67c17dad51fd610 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309291489e7e936e94b73a6609d582671a15e_0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:16:07,840 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023092924f37dd6a0ab49d0b0a27ef3a2510f7f_0f1c71e3f887ce47e67c17dad51fd610 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023092924f37dd6a0ab49d0b0a27ef3a2510f7f_0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:16:07,841 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309292f0817b7819e4db1b64fd742e7e4a2dd_0f1c71e3f887ce47e67c17dad51fd610 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309292f0817b7819e4db1b64fd742e7e4a2dd_0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:16:07,841 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309293aa3234a0283406fa5ea1d42bf6603a5_0f1c71e3f887ce47e67c17dad51fd610 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309293aa3234a0283406fa5ea1d42bf6603a5_0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:16:07,842 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309293f0bad0956294872937c9c847bcb2001_0f1c71e3f887ce47e67c17dad51fd610 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309293f0bad0956294872937c9c847bcb2001_0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:16:07,843 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929422894af6d12440aaa13aabc4fd90d3c_0f1c71e3f887ce47e67c17dad51fd610 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929422894af6d12440aaa13aabc4fd90d3c_0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:16:07,844 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929535266286e624c629ad2f96cdb84f4d8_0f1c71e3f887ce47e67c17dad51fd610 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929535266286e624c629ad2f96cdb84f4d8_0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:16:07,845 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309296325a3368b164108ae3672fc137ee5c4_0f1c71e3f887ce47e67c17dad51fd610 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309296325a3368b164108ae3672fc137ee5c4_0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:16:07,846 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309296b88976b21444dcbb2e0306c0f0845ab_0f1c71e3f887ce47e67c17dad51fd610 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309296b88976b21444dcbb2e0306c0f0845ab_0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:16:07,846 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929749424148688428694c6e2c61b027253_0f1c71e3f887ce47e67c17dad51fd610 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929749424148688428694c6e2c61b027253_0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:16:07,847 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309297e512fe23c6a4d4792fdcd3cce6e2627_0f1c71e3f887ce47e67c17dad51fd610 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309297e512fe23c6a4d4792fdcd3cce6e2627_0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:16:07,848 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309298249c67605e94dfca83fa39c237ca6d4_0f1c71e3f887ce47e67c17dad51fd610 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309298249c67605e94dfca83fa39c237ca6d4_0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:16:07,849 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023092989cba9d8e3af4801a20e81d0e580e252_0f1c71e3f887ce47e67c17dad51fd610 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023092989cba9d8e3af4801a20e81d0e580e252_0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:16:07,850 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309298a74f6092e704332aae6502ae3a1b621_0f1c71e3f887ce47e67c17dad51fd610 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309298a74f6092e704332aae6502ae3a1b621_0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:16:07,851 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023092990e5285d1fd249e9a5e5c685f8dc4d9c_0f1c71e3f887ce47e67c17dad51fd610 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2023092990e5285d1fd249e9a5e5c685f8dc4d9c_0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:16:07,851 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929966c67c204f64fce8a12863bcd47332b_0f1c71e3f887ce47e67c17dad51fd610 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929966c67c204f64fce8a12863bcd47332b_0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:16:07,852 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309299ad3a68b38fb4d479d49a8e7c2f5ce8e_0f1c71e3f887ce47e67c17dad51fd610 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309299ad3a68b38fb4d479d49a8e7c2f5ce8e_0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:16:07,853 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309299daee17f58a541fb9ef43dd1bd6d1f90_0f1c71e3f887ce47e67c17dad51fd610 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202309299daee17f58a541fb9ef43dd1bd6d1f90_0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:16:07,854 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929a23ecc42090c4e5a90d9c0a82e6c62a4_0f1c71e3f887ce47e67c17dad51fd610 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929a23ecc42090c4e5a90d9c0a82e6c62a4_0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:16:07,855 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929a24bf84f6a3e4e6386936b0997e6b0a4_0f1c71e3f887ce47e67c17dad51fd610 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929a24bf84f6a3e4e6386936b0997e6b0a4_0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:16:07,856 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929a78e13230b724511a035d6f36eb7958e_0f1c71e3f887ce47e67c17dad51fd610 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929a78e13230b724511a035d6f36eb7958e_0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:16:07,857 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929bc2e7d0761a84c1f867439dbc89f8611_0f1c71e3f887ce47e67c17dad51fd610 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929bc2e7d0761a84c1f867439dbc89f8611_0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:16:07,858 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929e5caab91836248caafbd178dd27eb743_0f1c71e3f887ce47e67c17dad51fd610 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929e5caab91836248caafbd178dd27eb743_0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:16:07,859 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929f12a498e4b61444185bb8e9711523086_0f1c71e3f887ce47e67c17dad51fd610 to hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20230929f12a498e4b61444185bb8e9711523086_0f1c71e3f887ce47e67c17dad51fd610 2023-09-29T06:16:07,859 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2023-09-29T06:16:07,861 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=197, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2023-09-29T06:16:07,863 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2023-09-29T06:16:07,864 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(387): Removing 'TestAcidGuarantees' descriptor. 2023-09-29T06:16:07,865 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=197, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2023-09-29T06:16:07,865 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(377): Removing 'TestAcidGuarantees' from region states. 2023-09-29T06:16:07,865 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1695968167865"}]},"ts":"9223372036854775807"} 2023-09-29T06:16:07,866 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-09-29T06:16:07,866 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 0f1c71e3f887ce47e67c17dad51fd610, NAME => 'TestAcidGuarantees,,1695968135087.0f1c71e3f887ce47e67c17dad51fd610.', STARTKEY => '', ENDKEY => ''}] 2023-09-29T06:16:07,866 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(381): Marking 'TestAcidGuarantees' as deleted. 2023-09-29T06:16:07,867 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1695968167866"}]},"ts":"9223372036854775807"} 2023-09-29T06:16:07,868 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1678): Deleted table TestAcidGuarantees state from META 2023-09-29T06:16:07,869 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(127): Finished pid=197, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2023-09-29T06:16:07,870 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=197, state=SUCCESS; DeleteTableProcedure table=TestAcidGuarantees in 55 msec 2023-09-29T06:16:07,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42173 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=197 2023-09-29T06:16:07,917 INFO [Listener at localhost/42843 {}] client.HBaseAdmin$TableFuture(3677): Operation: DELETE, Table Name: default:TestAcidGuarantees, procId: 197 completed 2023-09-29T06:16:07,929 INFO [Listener at localhost/42843 {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithAdaptivePolicy#testMobGetAtomicity Thread=219 (was 221), OpenFileDescriptor=560 (was 600), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=384 (was 505), ProcessCount=167 (was 167), AvailableMemoryMB=1395 (was 1395) 2023-09-29T06:16:07,929 INFO [Listener at localhost/42843 {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2023-09-29T06:16:07,929 INFO [Listener at localhost/42843 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-09-29T06:16:07,930 DEBUG [Listener at localhost/42843 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x3f46555d to 127.0.0.1:49344 2023-09-29T06:16:07,930 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:16:07,930 DEBUG [Listener at localhost/42843 {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2023-09-29T06:16:07,930 DEBUG [Listener at localhost/42843 {}] util.JVMClusterUtil(257): Found active master hash=519929528, stopped=false 2023-09-29T06:16:07,930 INFO [Listener at localhost/42843 {}] master.ServerManager(910): Cluster shutdown requested of master=jenkins-hbase3.apache.org,42173,1695967976351 2023-09-29T06:16:07,932 DEBUG [Listener at localhost/42843-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39783-0x101625af8180001, quorum=127.0.0.1:49344, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2023-09-29T06:16:07,932 DEBUG [Listener at localhost/42843-EventThread {}] zookeeper.ZKWatcher(604): master:42173-0x101625af8180000, quorum=127.0.0.1:49344, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2023-09-29T06:16:07,932 DEBUG [Listener at localhost/42843-EventThread {}] zookeeper.ZKWatcher(604): master:42173-0x101625af8180000, quorum=127.0.0.1:49344, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-09-29T06:16:07,932 DEBUG [Listener at localhost/42843-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39783-0x101625af8180001, quorum=127.0.0.1:49344, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-09-29T06:16:07,932 INFO [Listener at localhost/42843 {}] procedure2.ProcedureExecutor(630): Stopping 2023-09-29T06:16:07,933 DEBUG [Listener at localhost/42843 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:16:07,933 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42173-0x101625af8180000, quorum=127.0.0.1:49344, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2023-09-29T06:16:07,933 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39783-0x101625af8180001, quorum=127.0.0.1:49344, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2023-09-29T06:16:07,933 INFO [Listener at localhost/42843 {}] regionserver.HRegionServer(2517): ***** STOPPING region server 'jenkins-hbase3.apache.org,39783,1695967978138' ***** 2023-09-29T06:16:07,933 INFO [Listener at localhost/42843 {}] regionserver.HRegionServer(2531): STOPPED: Shutdown requested 2023-09-29T06:16:07,933 INFO [RS:0;jenkins-hbase3:39783 {}] regionserver.HeapMemoryManager(220): Stopping 2023-09-29T06:16:07,933 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2023-09-29T06:16:07,933 INFO [RS:0;jenkins-hbase3:39783 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2023-09-29T06:16:07,933 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2023-09-29T06:16:07,934 INFO [RS:0;jenkins-hbase3:39783 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2023-09-29T06:16:07,934 INFO [RS:0;jenkins-hbase3:39783 {}] regionserver.HRegionServer(3528): Received CLOSE for 013609be28cb78869a9efb7623dcb4f7 2023-09-29T06:16:07,934 INFO [RS:0;jenkins-hbase3:39783 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:16:07,934 DEBUG [RS:0;jenkins-hbase3:39783 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:16:07,934 INFO [RS:0;jenkins-hbase3:39783 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2023-09-29T06:16:07,934 INFO [RS:0;jenkins-hbase3:39783 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2023-09-29T06:16:07,934 INFO [RS:0;jenkins-hbase3:39783 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2023-09-29T06:16:07,934 INFO [RS:0;jenkins-hbase3:39783 {}] regionserver.HRegionServer(3528): Received CLOSE for 1588230740 2023-09-29T06:16:07,934 INFO [RS:0;jenkins-hbase3:39783 {}] regionserver.HRegionServer(1579): Waiting on 2 regions to close 2023-09-29T06:16:07,935 DEBUG [RS:0;jenkins-hbase3:39783 {}] regionserver.HRegionServer(1583): Online Regions={1588230740=hbase:meta,,1.1588230740, 013609be28cb78869a9efb7623dcb4f7=hbase:namespace,,1695967981879.013609be28cb78869a9efb7623dcb4f7.} 2023-09-29T06:16:07,935 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 013609be28cb78869a9efb7623dcb4f7, disabling compactions & flushes 2023-09-29T06:16:07,935 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:namespace,,1695967981879.013609be28cb78869a9efb7623dcb4f7. 2023-09-29T06:16:07,935 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1695967981879.013609be28cb78869a9efb7623dcb4f7. 2023-09-29T06:16:07,935 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-09-29T06:16:07,935 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1695967981879.013609be28cb78869a9efb7623dcb4f7. after waiting 0 ms 2023-09-29T06:16:07,935 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1695967981879.013609be28cb78869a9efb7623dcb4f7. 2023-09-29T06:16:07,935 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-09-29T06:16:07,935 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-09-29T06:16:07,935 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 013609be28cb78869a9efb7623dcb4f7 1/1 column families, dataSize=78 B heapSize=488 B 2023-09-29T06:16:07,935 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-09-29T06:16:07,935 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-09-29T06:16:07,938 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2819): Flushing 1588230740 3/3 column families, dataSize=20.92 KB heapSize=36.24 KB 2023-09-29T06:16:07,939 DEBUG [RS:0;jenkins-hbase3:39783 {}] regionserver.HRegionServer(1609): Waiting on 013609be28cb78869a9efb7623dcb4f7, 1588230740 2023-09-29T06:16:07,950 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/hbase/namespace/013609be28cb78869a9efb7623dcb4f7/.tmp/info/43d24aa889fa4a44811fd353029f2052 is 45, key is default/info:d/1695967982466/Put/seqid=0 2023-09-29T06:16:07,955 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/hbase/meta/1588230740/.tmp/info/dc69237eb4714ad3a1617dcd02ec8733 is 143, key is hbase:namespace,,1695967981879.013609be28cb78869a9efb7623dcb4f7./info:regioninfo/1695967982359/Put/seqid=0 2023-09-29T06:16:07,960 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=18.26 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/hbase/meta/1588230740/.tmp/info/dc69237eb4714ad3a1617dcd02ec8733 2023-09-29T06:16:07,975 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/hbase/meta/1588230740/.tmp/rep_barrier/989c935cf43a482eb7b8c01752bfb98e is 102, key is TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de./rep_barrier:/1695968012491/DeleteFamily/seqid=0 2023-09-29T06:16:07,977 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/hbase/meta/1588230740/.tmp/rep_barrier/989c935cf43a482eb7b8c01752bfb98e 2023-09-29T06:16:07,995 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/hbase/meta/1588230740/.tmp/table/666d8cc1803346f8a7800bb63a90aaa7 is 96, key is TestAcidGuarantees,,1695967982752.1291e331f06575ebce0d38fd8ca0d9de./table:/1695968012491/DeleteFamily/seqid=0 2023-09-29T06:16:08,079 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2023-09-29T06:16:08,079 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2023-09-29T06:16:08,139 DEBUG [RS:0;jenkins-hbase3:39783 {}] regionserver.HRegionServer(1609): Waiting on 013609be28cb78869a9efb7623dcb4f7, 1588230740 2023-09-29T06:16:08,339 DEBUG [RS:0;jenkins-hbase3:39783 {}] regionserver.HRegionServer(1609): Waiting on 013609be28cb78869a9efb7623dcb4f7, 1588230740 2023-09-29T06:16:08,353 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/hbase/namespace/013609be28cb78869a9efb7623dcb4f7/.tmp/info/43d24aa889fa4a44811fd353029f2052 2023-09-29T06:16:08,357 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/hbase/namespace/013609be28cb78869a9efb7623dcb4f7/.tmp/info/43d24aa889fa4a44811fd353029f2052 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/hbase/namespace/013609be28cb78869a9efb7623dcb4f7/info/43d24aa889fa4a44811fd353029f2052 2023-09-29T06:16:08,360 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/hbase/namespace/013609be28cb78869a9efb7623dcb4f7/info/43d24aa889fa4a44811fd353029f2052, entries=2, sequenceid=6, filesize=4.9 K 2023-09-29T06:16:08,361 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~78 B/78, heapSize ~472 B/472, currentSize=0 B/0 for 013609be28cb78869a9efb7623dcb4f7 in 426ms, sequenceid=6, compaction requested=false 2023-09-29T06:16:08,364 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/hbase/namespace/013609be28cb78869a9efb7623dcb4f7/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2023-09-29T06:16:08,365 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:namespace,,1695967981879.013609be28cb78869a9efb7623dcb4f7. 2023-09-29T06:16:08,365 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 013609be28cb78869a9efb7623dcb4f7: 2023-09-29T06:16:08,365 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1695967981879.013609be28cb78869a9efb7623dcb4f7. 2023-09-29T06:16:08,398 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.08 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/hbase/meta/1588230740/.tmp/table/666d8cc1803346f8a7800bb63a90aaa7 2023-09-29T06:16:08,401 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/hbase/meta/1588230740/.tmp/info/dc69237eb4714ad3a1617dcd02ec8733 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/hbase/meta/1588230740/info/dc69237eb4714ad3a1617dcd02ec8733 2023-09-29T06:16:08,403 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/hbase/meta/1588230740/info/dc69237eb4714ad3a1617dcd02ec8733, entries=22, sequenceid=93, filesize=7.6 K 2023-09-29T06:16:08,404 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/hbase/meta/1588230740/.tmp/rep_barrier/989c935cf43a482eb7b8c01752bfb98e as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/hbase/meta/1588230740/rep_barrier/989c935cf43a482eb7b8c01752bfb98e 2023-09-29T06:16:08,406 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/hbase/meta/1588230740/rep_barrier/989c935cf43a482eb7b8c01752bfb98e, entries=6, sequenceid=93, filesize=5.9 K 2023-09-29T06:16:08,407 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/hbase/meta/1588230740/.tmp/table/666d8cc1803346f8a7800bb63a90aaa7 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/hbase/meta/1588230740/table/666d8cc1803346f8a7800bb63a90aaa7 2023-09-29T06:16:08,409 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/hbase/meta/1588230740/table/666d8cc1803346f8a7800bb63a90aaa7, entries=9, sequenceid=93, filesize=5.8 K 2023-09-29T06:16:08,410 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3022): Finished flush of dataSize ~20.92 KB/21417, heapSize ~36.20 KB/37064, currentSize=0 B/0 for 1588230740 in 475ms, sequenceid=93, compaction requested=false 2023-09-29T06:16:08,413 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/data/hbase/meta/1588230740/recovered.edits/96.seqid, newMaxSeqId=96, maxSeqId=1 2023-09-29T06:16:08,414 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-09-29T06:16:08,415 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-09-29T06:16:08,415 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-09-29T06:16:08,415 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2023-09-29T06:16:08,540 INFO [RS:0;jenkins-hbase3:39783 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,39783,1695967978138; all regions closed. 2023-09-29T06:16:08,547 DEBUG [RS:0;jenkins-hbase3:39783 {}] wal.AbstractFSWAL(1062): Moved 1 WAL file(s) to /user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/oldWALs 2023-09-29T06:16:08,547 INFO [RS:0;jenkins-hbase3:39783 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C39783%2C1695967978138.meta:.meta(num 1695967981560) 2023-09-29T06:16:08,550 DEBUG [RS:0;jenkins-hbase3:39783 {}] wal.AbstractFSWAL(1062): Moved 1 WAL file(s) to /user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/oldWALs 2023-09-29T06:16:08,550 INFO [RS:0;jenkins-hbase3:39783 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C39783%2C1695967978138:(num 1695967981185) 2023-09-29T06:16:08,550 DEBUG [RS:0;jenkins-hbase3:39783 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:16:08,551 INFO [RS:0;jenkins-hbase3:39783 {}] regionserver.LeaseManager(133): Closed leases 2023-09-29T06:16:08,551 INFO [RS:0;jenkins-hbase3:39783 {}] hbase.ChoreService(370): Chore service for: regionserver/jenkins-hbase3:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS] on shutdown 2023-09-29T06:16:08,551 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-09-29T06:16:08,552 INFO [RS:0;jenkins-hbase3:39783 {}] ipc.NettyRpcServer(346): Stopping server on /172.31.12.81:39783 2023-09-29T06:16:08,558 DEBUG [Listener at localhost/42843-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39783-0x101625af8180001, quorum=127.0.0.1:49344, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/jenkins-hbase3.apache.org,39783,1695967978138 2023-09-29T06:16:08,558 DEBUG [Listener at localhost/42843-EventThread {}] zookeeper.ZKWatcher(604): master:42173-0x101625af8180000, quorum=127.0.0.1:49344, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2023-09-29T06:16:08,560 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [jenkins-hbase3.apache.org,39783,1695967978138] 2023-09-29T06:16:08,561 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing jenkins-hbase3.apache.org,39783,1695967978138; numProcessing=1 2023-09-29T06:16:08,562 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(224): Node /hbase/draining/jenkins-hbase3.apache.org,39783,1695967978138 already deleted, retry=false 2023-09-29T06:16:08,562 INFO [RegionServerTracker-0 {}] master.ServerManager(569): Cluster shutdown set; jenkins-hbase3.apache.org,39783,1695967978138 expired; onlineServers=0 2023-09-29T06:16:08,562 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2517): ***** STOPPING region server 'jenkins-hbase3.apache.org,42173,1695967976351' ***** 2023-09-29T06:16:08,562 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2531): STOPPED: Cluster shutdown set; onlineServer=0 2023-09-29T06:16:08,563 DEBUG [M:0;jenkins-hbase3:42173 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76d6e361, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-09-29T06:16:08,563 INFO [M:0;jenkins-hbase3:42173 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,42173,1695967976351 2023-09-29T06:16:08,563 INFO [M:0;jenkins-hbase3:42173 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,42173,1695967976351; all regions closed. 2023-09-29T06:16:08,563 DEBUG [M:0;jenkins-hbase3:42173 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-29T06:16:08,563 DEBUG [M:0;jenkins-hbase3:42173 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2023-09-29T06:16:08,563 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2023-09-29T06:16:08,563 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1695967980786 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1695967980786,5,FailOnTimeoutGroup] 2023-09-29T06:16:08,563 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1695967980787 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1695967980787,5,FailOnTimeoutGroup] 2023-09-29T06:16:08,563 DEBUG [M:0;jenkins-hbase3:42173 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2023-09-29T06:16:08,563 INFO [M:0;jenkins-hbase3:42173 {}] hbase.ChoreService(370): Chore service for: master/jenkins-hbase3:0 had [] on shutdown 2023-09-29T06:16:08,564 DEBUG [M:0;jenkins-hbase3:42173 {}] master.HMaster(1688): Stopping service threads 2023-09-29T06:16:08,564 INFO [M:0;jenkins-hbase3:42173 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2023-09-29T06:16:08,564 ERROR [M:0;jenkins-hbase3:42173 {}] procedure2.ProcedureExecutor(654): ThreadGroup java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] contains running threads; null: See STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (934951672) connection to localhost/127.0.0.1:34233 from jenkins,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2023-09-29T06:16:08,564 DEBUG [Listener at localhost/42843-EventThread {}] zookeeper.ZKWatcher(604): master:42173-0x101625af8180000, quorum=127.0.0.1:49344, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2023-09-29T06:16:08,564 DEBUG [Listener at localhost/42843-EventThread {}] zookeeper.ZKWatcher(604): master:42173-0x101625af8180000, quorum=127.0.0.1:49344, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-09-29T06:16:08,564 INFO [M:0;jenkins-hbase3:42173 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2023-09-29T06:16:08,564 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2023-09-29T06:16:08,565 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42173-0x101625af8180000, quorum=127.0.0.1:49344, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2023-09-29T06:16:08,565 DEBUG [M:0;jenkins-hbase3:42173 {}] zookeeper.ZKUtil(347): master:42173-0x101625af8180000, quorum=127.0.0.1:49344, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2023-09-29T06:16:08,565 WARN [M:0;jenkins-hbase3:42173 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2023-09-29T06:16:08,565 INFO [M:0;jenkins-hbase3:42173 {}] assignment.AssignmentManager(382): Stopping assignment manager 2023-09-29T06:16:08,565 INFO [M:0;jenkins-hbase3:42173 {}] region.MasterRegion(178): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2023-09-29T06:16:08,566 DEBUG [M:0;jenkins-hbase3:42173 {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-09-29T06:16:08,566 INFO [M:0;jenkins-hbase3:42173 {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-29T06:16:08,566 DEBUG [M:0;jenkins-hbase3:42173 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-29T06:16:08,566 DEBUG [M:0;jenkins-hbase3:42173 {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-09-29T06:16:08,566 DEBUG [M:0;jenkins-hbase3:42173 {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-29T06:16:08,566 INFO [M:0;jenkins-hbase3:42173 {}] regionserver.HRegion(2819): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=807.65 KB heapSize=994.30 KB 2023-09-29T06:16:08,577 DEBUG [M:0;jenkins-hbase3:42173 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4ee1a94a09ee4a979215288f4ad68bf0 is 88, key is hbase:meta,,1/info:sn/1695967981310/Put/seqid=0 2023-09-29T06:16:08,580 INFO [M:0;jenkins-hbase3:42173 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=530 B at sequenceid=2297 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4ee1a94a09ee4a979215288f4ad68bf0 2023-09-29T06:16:08,595 DEBUG [M:0;jenkins-hbase3:42173 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7d44d753b693477c9eff6c60c4b273b1 is 2278, key is \x00\x00\x00\x00\x00\x00\x00f/proc:d/1695968081469/Put/seqid=0 2023-09-29T06:16:08,599 INFO [M:0;jenkins-hbase3:42173 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=807.06 KB at sequenceid=2297 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7d44d753b693477c9eff6c60c4b273b1 2023-09-29T06:16:08,601 INFO [M:0;jenkins-hbase3:42173 {}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7d44d753b693477c9eff6c60c4b273b1 2023-09-29T06:16:08,612 DEBUG [M:0;jenkins-hbase3:42173 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/150ce52afd88495e83b08fc6c18b8f78 is 82, key is jenkins-hbase3.apache.org,39783,1695967978138/rs:state/1695967980918/Put/seqid=0 2023-09-29T06:16:08,661 DEBUG [Listener at localhost/42843-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39783-0x101625af8180001, quorum=127.0.0.1:49344, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2023-09-29T06:16:08,661 INFO [RS:0;jenkins-hbase3:39783 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,39783,1695967978138; zookeeper connection closed. 2023-09-29T06:16:08,661 DEBUG [Listener at localhost/42843-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39783-0x101625af8180001, quorum=127.0.0.1:49344, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2023-09-29T06:16:08,661 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@558b5e06 {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@558b5e06 2023-09-29T06:16:08,662 INFO [Listener at localhost/42843 {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2023-09-29T06:16:09,016 INFO [M:0;jenkins-hbase3:42173 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=2297 (bloomFilter=true), to=hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/150ce52afd88495e83b08fc6c18b8f78 2023-09-29T06:16:09,020 DEBUG [M:0;jenkins-hbase3:42173 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4ee1a94a09ee4a979215288f4ad68bf0 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/4ee1a94a09ee4a979215288f4ad68bf0 2023-09-29T06:16:09,023 INFO [M:0;jenkins-hbase3:42173 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/4ee1a94a09ee4a979215288f4ad68bf0, entries=8, sequenceid=2297, filesize=5.5 K 2023-09-29T06:16:09,023 DEBUG [M:0;jenkins-hbase3:42173 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7d44d753b693477c9eff6c60c4b273b1 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/7d44d753b693477c9eff6c60c4b273b1 2023-09-29T06:16:09,026 INFO [M:0;jenkins-hbase3:42173 {}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7d44d753b693477c9eff6c60c4b273b1 2023-09-29T06:16:09,026 INFO [M:0;jenkins-hbase3:42173 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/7d44d753b693477c9eff6c60c4b273b1, entries=197, sequenceid=2297, filesize=48.0 K 2023-09-29T06:16:09,027 DEBUG [M:0;jenkins-hbase3:42173 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/150ce52afd88495e83b08fc6c18b8f78 as hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/150ce52afd88495e83b08fc6c18b8f78 2023-09-29T06:16:09,029 INFO [M:0;jenkins-hbase3:42173 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34233/user/jenkins/test-data/3c9207d3-079b-e555-411b-658cc0ffb41a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/150ce52afd88495e83b08fc6c18b8f78, entries=1, sequenceid=2297, filesize=5.1 K 2023-09-29T06:16:09,029 INFO [M:0;jenkins-hbase3:42173 {}] regionserver.HRegion(3022): Finished flush of dataSize ~807.65 KB/827034, heapSize ~994.01 KB/1017864, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 463ms, sequenceid=2297, compaction requested=false 2023-09-29T06:16:09,030 INFO [M:0;jenkins-hbase3:42173 {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-29T06:16:09,031 DEBUG [M:0;jenkins-hbase3:42173 {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-09-29T06:16:09,034 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-09-29T06:16:09,034 INFO [M:0;jenkins-hbase3:42173 {}] flush.MasterFlushTableProcedureManager(85): stop: server shutting down. 2023-09-29T06:16:09,035 INFO [M:0;jenkins-hbase3:42173 {}] ipc.NettyRpcServer(346): Stopping server on /172.31.12.81:42173 2023-09-29T06:16:09,037 DEBUG [M:0;jenkins-hbase3:42173 {}] zookeeper.RecoverableZooKeeper(224): Node /hbase/rs/jenkins-hbase3.apache.org,42173,1695967976351 already deleted, retry=false 2023-09-29T06:16:09,139 DEBUG [Listener at localhost/42843-EventThread {}] zookeeper.ZKWatcher(604): master:42173-0x101625af8180000, quorum=127.0.0.1:49344, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2023-09-29T06:16:09,139 INFO [M:0;jenkins-hbase3:42173 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,42173,1695967976351; zookeeper connection closed. 2023-09-29T06:16:09,139 DEBUG [Listener at localhost/42843-EventThread {}] zookeeper.ZKWatcher(604): master:42173-0x101625af8180000, quorum=127.0.0.1:49344, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2023-09-29T06:16:09,141 WARN [Listener at localhost/42843 {}] datanode.DirectoryScanner(537): DirectoryScanner: shutdown has been called 2023-09-29T06:16:09,145 INFO [Listener at localhost/42843 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-09-29T06:16:09,253 WARN [BP-771464227-172.31.12.81-1695967972816 heartbeating to localhost/127.0.0.1:34233 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2023-09-29T06:16:09,253 WARN [BP-771464227-172.31.12.81-1695967972816 heartbeating to localhost/127.0.0.1:34233 {}] datanode.BPServiceActor(862): Ending block pool service for: Block pool BP-771464227-172.31.12.81-1695967972816 (Datanode Uuid 3d6298e7-fdaf-4ed5-963c-adc3ebb4292e) service to localhost/127.0.0.1:34233 2023-09-29T06:16:09,255 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a22ba870-907c-9d99-3f64-a164cf019894/cluster_925cee83-985f-a2ad-bbe0-74639e9c7977/dfs/data/data1/current/BP-771464227-172.31.12.81-1695967972816 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-09-29T06:16:09,255 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a22ba870-907c-9d99-3f64-a164cf019894/cluster_925cee83-985f-a2ad-bbe0-74639e9c7977/dfs/data/data2/current/BP-771464227-172.31.12.81-1695967972816 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-09-29T06:16:09,293 INFO [Listener at localhost/42843 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-09-29T06:16:09,458 INFO [Listener at localhost/42843 {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers 2023-09-29T06:16:09,632 INFO [Listener at localhost/42843 {}] hbase.HBaseTestingUtility(1347): Minicluster is down